Use slavelog cache in notebooks (only pllld_count)

This commit is contained in:
Wataru Otsubo 2025-01-25 09:49:01 +01:00
parent d824f2c938
commit 2bab455089
4 changed files with 115 additions and 2126 deletions

View file

@ -1,8 +1,8 @@
# This file is machine-generated - editing it directly is not advised # This file is machine-generated - editing it directly is not advised
julia_version = "1.11.2" julia_version = "1.11.3"
manifest_format = "2.0" manifest_format = "2.0"
project_hash = "9bfd3227044b739391f5fee603ddc4de538f1c94" project_hash = "a5287f1b860b464c05770978a0adc638b6272c1f"
[[deps.ANSIColoredPrinters]] [[deps.ANSIColoredPrinters]]
git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"
@ -347,9 +347,9 @@ version = "6.1.2+0"
[[deps.FFTW]] [[deps.FFTW]]
deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"]
git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" git-tree-sha1 = "7de7c78d681078f027389e067864a8d53bd7c3c9"
uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
version = "1.8.0" version = "1.8.1"
[[deps.FFTW_jll]] [[deps.FFTW_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@ -543,9 +543,9 @@ version = "8.5.0+0"
[[deps.HypergeometricFunctions]] [[deps.HypergeometricFunctions]]
deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"]
git-tree-sha1 = "b1c2585431c382e3fe5805874bda6aea90a95de9" git-tree-sha1 = "2bd56245074fab4015b9174f24ceba8293209053"
uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
version = "0.3.25" version = "0.3.27"
[[deps.Hyperscript]] [[deps.Hyperscript]]
deps = ["Test"] deps = ["Test"]
@ -626,9 +626,9 @@ version = "1.4.2"
[[deps.IntelOpenMP_jll]] [[deps.IntelOpenMP_jll]]
deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"]
git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e" git-tree-sha1 = "0f14a5456bdc6b9731a5682f439a672750a09e48"
uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0"
version = "2024.2.1+0" version = "2025.0.4+0"
[[deps.InteractiveUtils]] [[deps.InteractiveUtils]]
deps = ["Markdown"] deps = ["Markdown"]
@ -710,6 +710,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d" uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0" version = "1.0.0"
[[deps.JLD2]]
deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"]
git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.5.11"
[[deps.JLLWrappers]] [[deps.JLLWrappers]]
deps = ["Artifacts", "Preferences"] deps = ["Artifacts", "Preferences"]
git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6"
@ -893,9 +899,9 @@ version = "0.1.4"
[[deps.MKL_jll]] [[deps.MKL_jll]]
deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"]
git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f" git-tree-sha1 = "5de60bc6cb3899cd318d80d627560fae2e2d99ae"
uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
version = "2024.2.0+0" version = "2025.0.1+1"
[[deps.MacroTools]] [[deps.MacroTools]]
git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472" git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472"
@ -983,9 +989,9 @@ version = "2023.12.12"
[[deps.NaNMath]] [[deps.NaNMath]]
deps = ["OpenLibm_jll"] deps = ["OpenLibm_jll"]
git-tree-sha1 = "030ea22804ef91648f29b7ad3fc15fa49d0e6e71" git-tree-sha1 = "fe891aea7ccd23897520db7f16931212454e277e"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "1.0.3" version = "1.1.1"
[[deps.Netpbm]] [[deps.Netpbm]]
deps = ["FileIO", "ImageCore", "ImageMetadata"] deps = ["FileIO", "ImageCore", "ImageMetadata"]
@ -1080,10 +1086,10 @@ uuid = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883"
version = "0.4.3" version = "0.4.3"
[[deps.PSBoardDataBase]] [[deps.PSBoardDataBase]]
deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "Printf", "SQLite", "StaticArrays", "Tables"] deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "JLD2", "Printf", "SQLite", "StaticArrays", "Tables"]
path = ".." path = ".."
uuid = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5" uuid = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5"
version = "0.4.0" version = "0.5.1"
weakdeps = ["InteractiveUtils"] weakdeps = ["InteractiveUtils"]
[deps.PSBoardDataBase.extensions] [deps.PSBoardDataBase.extensions]

View file

@ -5,6 +5,7 @@ DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
FHist = "68837c9b-b678-4cd5-9925-8a54edc8f695" FHist = "68837c9b-b678-4cd5-9925-8a54edc8f695"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
PSBoardDataBase = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5" PSBoardDataBase = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5"
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8" PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
@ -13,3 +14,6 @@ SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
[compat]
FHist = "0.11"

View file

@ -11,6 +11,7 @@ begin
using PSBoardDataBase using PSBoardDataBase
using SQLite using SQLite
using JLD2
using DataFrames using DataFrames
using CSV using CSV
using CairoMakie using CairoMakie
@ -40,6 +41,9 @@ db = SQLite.DB("../psboard_qaqc.db")
# ╔═╡ df5087be-18de-4714-b6d8-4ab54271b1fb # ╔═╡ df5087be-18de-4714-b6d8-4ab54271b1fb
SQLite.tables(db) SQLite.tables(db)
# ╔═╡ 566d3472-bd2d-489d-8736-857cb7ccaa93
jld2_slavelogs = jldopen("../slavelog.jld2")
# ╔═╡ 56d8896a-40e3-46e3-aa28-82f89476743b # ╔═╡ 56d8896a-40e3-46e3-aa28-82f89476743b
qaqc_runs = DBInterface.execute(db, "select * from qaqc_runs") |> DataFrame qaqc_runs = DBInterface.execute(db, "select * from qaqc_runs") |> DataFrame
@ -71,7 +75,11 @@ map(eachrow(qaqc_extra_run_campaign_6)) do row
@info "file $(logfile) not found" @info "file $(logfile) not found"
return missing return missing
end end
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile) slave_result = if haskey(jld2_slavelogs, logfilename)
jld2_slavelogs[logfilename]
else
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
end
map(slave_result.asdtp) do asdtp_result map(slave_result.asdtp) do asdtp_result
asdtp_result.pllld_fail_counter asdtp_result.pllld_fail_counter
end end
@ -90,8 +98,11 @@ df_pllld_counts_camp6 = combine(
@warn "logfile $(logfile) not found" @warn "logfile $(logfile) not found"
return [] return []
end end
slave_result = slave_result = if haskey(jld2_slavelogs, logfilename)
jld2_slavelogs[logfilename]
else
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile) PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
end
return map(slave_result.asdtp) do asdtp_result return map(slave_result.asdtp) do asdtp_result
( (
psboard_id = sdf.psboard_id[1], psboard_id = sdf.psboard_id[1],
@ -184,115 +195,66 @@ combine(
) )
# ╔═╡ a0baf37c-8b05-49c1-a6a4-027f3404f8b4 # ╔═╡ a0baf37c-8b05-49c1-a6a4-027f3404f8b4
df_reset_failed_though_reconfig_done = transform( @time df_reset_failed_though_reconfig_done = let
qaqc_extra_run_results, broken_list = filter(:is_slavelog_valid => !=(1), qaqc_extra_run_results)
[:psboard_id, :runid] => transform(
ByRow( qaqc_extra_run_results,
(psboard_id, runid) -> begin [:psboard_id, :runid] =>
broken_list = @NamedTuple{psbid::Int64, runid::Int64}[ ByRow(
(psbid = 1050, runid = 385), (psboard_id, runid) -> begin
(psbid = 1113, runid = 392), if psboard_id in broken_list.psboard_id && runid in broken_list.runid
(psbid = 1121, runid = 392), return (;
(psbid = 131, runid = 38), psboard_id,
(psbid = 133, runid = 38), runid,
(psbid = 136, runid = 38), reset_failed_though_reconfig_done = missing,
(psbid = 137, runid = 38), )
(psbid = 1545, runid = 474), end
(psbid = 16959, runid = 48),
(psbid = 16959, runid = 59),
(psbid = 198, runid = 251),
(psbid = 202, runid = 45),
(psbid = 203, runid = 45),
(psbid = 205, runid = 45),
(psbid = 206, runid = 45),
(psbid = 206, runid = 57),
(psbid = 206, runid = 59),
(psbid = 208, runid = 45),
(psbid = 210, runid = 45),
(psbid = 214, runid = 45),
(psbid = 215, runid = 48),
(psbid = 221, runid = 55),
(psbid = 223, runid = 48),
(psbid = 224, runid = 48),
(psbid = 229, runid = 48),
(psbid = 231, runid = 48),
(psbid = 234, runid = 48),
(psbid = 235, runid = 48),
(psbid = 237, runid = 45),
(psbid = 238, runid = 45),
(psbid = 239, runid = 45),
(psbid = 263, runid = 69),
(psbid = 329, runid = 69),
(psbid = 339, runid = 69),
(psbid = 42, runid = 178),
(psbid = 488791279, runid = 63),
(psbid = 48879, runid = 47),
(psbid = 723, runid = 436),
]
if (; psbid = psboard_id, runid) in broken_list
@error "broken file psbid: $(psboard_id), runid: $(runid)"
return (;
psboard_id,
runid,
reset_failed_though_reconfig_done = missing,
)
end
logfilename = "$(psboard_id)_$(runid)_longrun.txt" if runid == 99
logfile = joinpath("../test/input/slavelogs/main/", logfilename) @info "skipping runid 99 since it is not exactly 100 run"
!ispath(logfile) && begin return (
@warn "logfile $(logfile) not found" psboard_id = psboard_id,
runid = runid,
reset_failed_though_reconfig_done = missing,
)
end
logfilename = "$(psboard_id)_$(runid)_longrun.txt"
slave_result = if haskey(jld2_slavelogs, logfilename)
jld2_slavelogs[logfilename]
else
logfile = joinpath("../test/input/slavelogs/main/", logfilename)
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
end
return ( return (
psboard_id = psboard_id, psboard_id = psboard_id,
runid = runid, runid = runid,
reset_failed_though_reconfig_done = missing, reset_failed_though_reconfig_done = count(
slave_result.asdtp,
) do asdtp_result
asdtp_result.reconfig_done_2 != asdtp_result.reconfig_done
end,
) )
end end,
if runid == 99 ) => [:psboard_id, :runid, :reset_failed_though_reconfig_done],
@info "skipping runid 99 since it is not exactly 100 run" )
return ( end
psboard_id = psboard_id,
runid = runid,
reset_failed_though_reconfig_done = missing,
)
end
if psboard_id == 1545 && runid == 474
@info "skipping psbid 1545 runid 474 since it is broken"
return (
psboard_id = psboard_id,
runid = runid,
reset_failed_though_reconfig_done = missing,
)
end
slave_result = try
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
catch e
@error "Failed to parse slavelog file: $(logfile)"
throw(e)
end
return (
psboard_id = psboard_id,
runid = runid,
reset_failed_though_reconfig_done = count(
slave_result.asdtp,
) do asdtp_result
asdtp_result.reconfig_done_2 != asdtp_result.reconfig_done
end,
)
end,
) => [:psboard_id, :runid, :reset_failed_though_reconfig_done],
)
# ╔═╡ 20633b30-a703-47bc-92a6-8743abbd375c # ╔═╡ 20633b30-a703-47bc-92a6-8743abbd375c
sort(dropmissing(df_reset_failed_though_reconfig_done, :reset_failed_though_reconfig_done), :reset_failed_though_reconfig_done, rev = true) sort(
dropmissing(df_reset_failed_though_reconfig_done, :reset_failed_though_reconfig_done),
:reset_failed_though_reconfig_done,
rev = true,
)
# ╔═╡ 9cf8e7cb-a1f6-4c5b-8c3b-3872d2020e57 # ╔═╡ 9cf8e7cb-a1f6-4c5b-8c3b-3872d2020e57
result = result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
PSBoardDataBase.SlaveLogParser.parse_slavelog_file("../test/input/slavelogs/main/1070_310_longrun.txt") "../test/input/slavelogs/main/1070_310_longrun.txt",
)
# ╔═╡ 9b055b29-cf54-457d-8776-39430554ef9f # ╔═╡ 9b055b29-cf54-457d-8776-39430554ef9f
findall(result.asdtp) do result findall(result.asdtp) do result
result.reconfig_done != result.reconfig_done_2 result.reconfig_done != result.reconfig_done_2
end end
# ╔═╡ d5d479b5-4d55-4a55-aa5f-893141981838 # ╔═╡ d5d479b5-4d55-4a55-aa5f-893141981838
@ -309,7 +271,13 @@ let
end end
# ╔═╡ dfeb3900-539f-4d8d-8103-edb67cf4e890 # ╔═╡ dfeb3900-539f-4d8d-8103-edb67cf4e890
CSV.write("reset_failed_though_reconfig_done.csv", df_reset_failed_though_reconfig_done[:, [:psboard_id, :runid, :reset_failed_though_reconfig_done]]) # CSV.write(
# "reset_failed_though_reconfig_done.csv",
# df_reset_failed_though_reconfig_done[
# :,
# [:psboard_id, :runid, :reset_failed_though_reconfig_done],
# ],
# )
# ╔═╡ 332b7481-f6dc-4027-a5c5-cf58d5b6acd3 # ╔═╡ 332b7481-f6dc-4027-a5c5-cf58d5b6acd3
md""" md"""
@ -325,12 +293,17 @@ asdtp_result_total_fail_vs_ppasic = let
asdtp_result_total_vs_ppasic = zeros(Int64, 8) asdtp_result_total_vs_ppasic = zeros(Int64, 8)
for row in eachrow(df) for row in eachrow(df)
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file( slave_result =
joinpath( if haskey(jld2_slavelogs, "$(row.psboard_id)_$(row.runid)_longrun.txt")
"../test/input/slavelogs/main/", jld2_slavelogs["$(row.psboard_id)_$(row.runid)_longrun.txt"]
"$(row.psboard_id)_$(row.runid)_longrun.txt", else
), PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
) joinpath(
"../test/input/slavelogs/main/",
"$(row.psboard_id)_$(row.runid)_longrun.txt",
),
)
end
asdtp_results = slave_result.asdtp asdtp_results = slave_result.asdtp
@assert length(asdtp_results) == 100 @assert length(asdtp_results) == 100
# @info "" first(asdtp_results) # @info "" first(asdtp_results)
@ -383,12 +356,17 @@ asdtp_result_total_pp7_vs_ch = let
asdtp_result_total_pp7_vs_ch = zeros(Int64, 32) asdtp_result_total_pp7_vs_ch = zeros(Int64, 32)
for row in eachrow(df) for row in eachrow(df)
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file( slave_result =
joinpath( if haskey(jld2_slavelogs, "$(row.psboard_id)_$(row.runid)_longrun.txt")
"../test/input/slavelogs/main/", jld2_slavelogs["$(row.psboard_id)_$(row.runid)_longrun.txt"]
"$(row.psboard_id)_$(row.runid)_longrun.txt", else
), PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
) joinpath(
"../test/input/slavelogs/main/",
"$(row.psboard_id)_$(row.runid)_longrun.txt",
),
)
end
asdtp_results = slave_result.asdtp asdtp_results = slave_result.asdtp
@assert length(asdtp_results) == 100 @assert length(asdtp_results) == 100
# @info "" first(asdtp_results) # @info "" first(asdtp_results)
@ -434,6 +412,7 @@ barplot(
# ╟─26954da1-0820-490d-aa3e-79fd3603d28a # ╟─26954da1-0820-490d-aa3e-79fd3603d28a
# ╠═a36886af-ff32-4f4f-a20f-f83bc333b8fd # ╠═a36886af-ff32-4f4f-a20f-f83bc333b8fd
# ╠═df5087be-18de-4714-b6d8-4ab54271b1fb # ╠═df5087be-18de-4714-b6d8-4ab54271b1fb
# ╠═566d3472-bd2d-489d-8736-857cb7ccaa93
# ╠═56d8896a-40e3-46e3-aa28-82f89476743b # ╠═56d8896a-40e3-46e3-aa28-82f89476743b
# ╠═08032250-5893-445c-8662-459e5636ba81 # ╠═08032250-5893-445c-8662-459e5636ba81
# ╠═013948ff-ccde-4222-862b-7ecda62fd7f1 # ╠═013948ff-ccde-4222-862b-7ecda62fd7f1

File diff suppressed because it is too large Load diff