research/PSBoardDataBase
1
0
Fork 0
mirror of https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git synced 2025-06-08 05:55:42 +09:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use slavelog cache in notebooks (only pllld_count)

Browse source
This commit is contained in:
Wataru Otsubo 2025-01-25 09:49:01 +01:00
parent d824f2c938
commit 2bab455089
4 changed files with 115 additions and 2126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
examples/Manifest.toml
View file

@ -1,8 +1,8 @@
# This file is machine-generated - editing it directly is not advised
julia_version = "1.11.2"
julia_version = "1.11.3"
manifest_format = "2.0"
project_hash = "9bfd3227044b739391f5fee603ddc4de538f1c94"
project_hash = "a5287f1b860b464c05770978a0adc638b6272c1f"
[[deps.ANSIColoredPrinters]]
git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"
@ -347,9 +347,9 @@ version = "6.1.2+0"
[[deps.FFTW]]
deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"]
git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d"
git-tree-sha1 = "7de7c78d681078f027389e067864a8d53bd7c3c9"
uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
version = "1.8.0"
version = "1.8.1"
[[deps.FFTW_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@ -543,9 +543,9 @@ version = "8.5.0+0"
[[deps.HypergeometricFunctions]]
deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"]
git-tree-sha1 = "b1c2585431c382e3fe5805874bda6aea90a95de9"
git-tree-sha1 = "2bd56245074fab4015b9174f24ceba8293209053"
uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
version = "0.3.25"
version = "0.3.27"
[[deps.Hyperscript]]
deps = ["Test"]
@ -626,9 +626,9 @@ version = "1.4.2"
[[deps.IntelOpenMP_jll]]
deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"]
git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e"
git-tree-sha1 = "0f14a5456bdc6b9731a5682f439a672750a09e48"
uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0"
version = "2024.2.1+0"
version = "2025.0.4+0"
[[deps.InteractiveUtils]]
deps = ["Markdown"]
@ -710,6 +710,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"
[[deps.JLD2]]
deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"]
git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.5.11"
[[deps.JLLWrappers]]
deps = ["Artifacts", "Preferences"]
git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6"
@ -893,9 +899,9 @@ version = "0.1.4"
[[deps.MKL_jll]]
deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"]
git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f"
git-tree-sha1 = "5de60bc6cb3899cd318d80d627560fae2e2d99ae"
uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
version = "2024.2.0+0"
version = "2025.0.1+1"
[[deps.MacroTools]]
git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472"
@ -983,9 +989,9 @@ version = "2023.12.12"
[[deps.NaNMath]]
deps = ["OpenLibm_jll"]
git-tree-sha1 = "030ea22804ef91648f29b7ad3fc15fa49d0e6e71"
git-tree-sha1 = "fe891aea7ccd23897520db7f16931212454e277e"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "1.0.3"
version = "1.1.1"
[[deps.Netpbm]]
deps = ["FileIO", "ImageCore", "ImageMetadata"]
@ -1080,10 +1086,10 @@ uuid = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883"
version = "0.4.3"
[[deps.PSBoardDataBase]]
deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "Printf", "SQLite", "StaticArrays", "Tables"]
deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "JLD2", "Printf", "SQLite", "StaticArrays", "Tables"]
path = ".."
uuid = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5"
version = "0.4.0"
version = "0.5.1"
weakdeps = ["InteractiveUtils"]
[deps.PSBoardDataBase.extensions]

4
examples/Project.toml
View file

@ -5,6 +5,7 @@ DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
FHist = "68837c9b-b678-4cd5-9925-8a54edc8f695"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
PSBoardDataBase = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5"
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
@ -13,3 +14,6 @@ SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
[compat]
FHist = "0.11"

127
examples/pllld_count.jl
View file

@ -11,6 +11,7 @@ begin
using PSBoardDataBase
using SQLite
using JLD2
using DataFrames
using CSV
using CairoMakie
@ -40,6 +41,9 @@ db = SQLite.DB("../psboard_qaqc.db")
# ╔═╡ df5087be-18de-4714-b6d8-4ab54271b1fb
SQLite.tables(db)
# ╔═╡ 566d3472-bd2d-489d-8736-857cb7ccaa93
jld2_slavelogs = jldopen("../slavelog.jld2")
# ╔═╡ 56d8896a-40e3-46e3-aa28-82f89476743b
qaqc_runs = DBInterface.execute(db, "select * from qaqc_runs") |> DataFrame
@ -71,7 +75,11 @@ map(eachrow(qaqc_extra_run_campaign_6)) do row
@info "file $(logfile) not found"
return missing
end
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
slave_result = if haskey(jld2_slavelogs, logfilename)
jld2_slavelogs[logfilename]
else
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
end
map(slave_result.asdtp) do asdtp_result
asdtp_result.pllld_fail_counter
end
@ -90,8 +98,11 @@ df_pllld_counts_camp6 = combine(
@warn "logfile $(logfile) not found"
return []
end
slave_result =
slave_result = if haskey(jld2_slavelogs, logfilename)
jld2_slavelogs[logfilename]
else
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
end
return map(slave_result.asdtp) do asdtp_result
(
psboard_id = sdf.psboard_id[1],
@ -184,53 +195,14 @@ combine(
)
# ╔═╡ a0baf37c-8b05-49c1-a6a4-027f3404f8b4
df_reset_failed_though_reconfig_done = transform(
@time df_reset_failed_though_reconfig_done = let
broken_list = filter(:is_slavelog_valid => !=(1), qaqc_extra_run_results)
transform(
qaqc_extra_run_results,
[:psboard_id, :runid] =>
ByRow(
(psboard_id, runid) -> begin
broken_list = @NamedTuple{psbid::Int64, runid::Int64}[
(psbid = 1050, runid = 385),
(psbid = 1113, runid = 392),
(psbid = 1121, runid = 392),
(psbid = 131, runid = 38),
(psbid = 133, runid = 38),
(psbid = 136, runid = 38),
(psbid = 137, runid = 38),
(psbid = 1545, runid = 474),
(psbid = 16959, runid = 48),
(psbid = 16959, runid = 59),
(psbid = 198, runid = 251),
(psbid = 202, runid = 45),
(psbid = 203, runid = 45),
(psbid = 205, runid = 45),
(psbid = 206, runid = 45),
(psbid = 206, runid = 57),
(psbid = 206, runid = 59),
(psbid = 208, runid = 45),
(psbid = 210, runid = 45),
(psbid = 214, runid = 45),
(psbid = 215, runid = 48),
(psbid = 221, runid = 55),
(psbid = 223, runid = 48),
(psbid = 224, runid = 48),
(psbid = 229, runid = 48),
(psbid = 231, runid = 48),
(psbid = 234, runid = 48),
(psbid = 235, runid = 48),
(psbid = 237, runid = 45),
(psbid = 238, runid = 45),
(psbid = 239, runid = 45),
(psbid = 263, runid = 69),
(psbid = 329, runid = 69),
(psbid = 339, runid = 69),
(psbid = 42, runid = 178),
(psbid = 488791279, runid = 63),
(psbid = 48879, runid = 47),
(psbid = 723, runid = 436),
]
if (; psbid = psboard_id, runid) in broken_list
@error "broken file psbid: $(psboard_id), runid: $(runid)"
if psboard_id in broken_list.psboard_id && runid in broken_list.runid
return (;
psboard_id,
runid,
@ -238,16 +210,6 @@ df_reset_failed_though_reconfig_done = transform(
)
end
logfilename = "$(psboard_id)_$(runid)_longrun.txt"
logfile = joinpath("../test/input/slavelogs/main/", logfilename)
!ispath(logfile) && begin
@warn "logfile $(logfile) not found"
return (
psboard_id = psboard_id,
runid = runid,
reset_failed_though_reconfig_done = missing,
)
end
if runid == 99
@info "skipping runid 99 since it is not exactly 100 run"
return (
@ -256,19 +218,13 @@ df_reset_failed_though_reconfig_done = transform(
reset_failed_though_reconfig_done = missing,
)
end
if psboard_id == 1545 && runid == 474
@info "skipping psbid 1545 runid 474 since it is broken"
return (
psboard_id = psboard_id,
runid = runid,
reset_failed_though_reconfig_done = missing,
)
end
slave_result = try
logfilename = "$(psboard_id)_$(runid)_longrun.txt"
slave_result = if haskey(jld2_slavelogs, logfilename)
jld2_slavelogs[logfilename]
else
logfile = joinpath("../test/input/slavelogs/main/", logfilename)
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
catch e
@error "Failed to parse slavelog file: $(logfile)"
throw(e)
end
return (
psboard_id = psboard_id,
@ -281,14 +237,20 @@ df_reset_failed_though_reconfig_done = transform(
)
end,
) => [:psboard_id, :runid, :reset_failed_though_reconfig_done],
)
)
end
# ╔═╡ 20633b30-a703-47bc-92a6-8743abbd375c
sort(dropmissing(df_reset_failed_though_reconfig_done, :reset_failed_though_reconfig_done), :reset_failed_though_reconfig_done, rev = true)
sort(
dropmissing(df_reset_failed_though_reconfig_done, :reset_failed_though_reconfig_done),
:reset_failed_though_reconfig_done,
rev = true,
)
# ╔═╡ 9cf8e7cb-a1f6-4c5b-8c3b-3872d2020e57
result =
PSBoardDataBase.SlaveLogParser.parse_slavelog_file("../test/input/slavelogs/main/1070_310_longrun.txt")
result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
"../test/input/slavelogs/main/1070_310_longrun.txt",
)
# ╔═╡ 9b055b29-cf54-457d-8776-39430554ef9f
findall(result.asdtp) do result
@ -309,7 +271,13 @@ let
end
# ╔═╡ dfeb3900-539f-4d8d-8103-edb67cf4e890
CSV.write("reset_failed_though_reconfig_done.csv", df_reset_failed_though_reconfig_done[:, [:psboard_id, :runid, :reset_failed_though_reconfig_done]])
# CSV.write(
# "reset_failed_though_reconfig_done.csv",
# df_reset_failed_though_reconfig_done[
# :,
# [:psboard_id, :runid, :reset_failed_though_reconfig_done],
# ],
# )
# ╔═╡ 332b7481-f6dc-4027-a5c5-cf58d5b6acd3
md"""
@ -325,12 +293,17 @@ asdtp_result_total_fail_vs_ppasic = let
asdtp_result_total_vs_ppasic = zeros(Int64, 8)
for row in eachrow(df)
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
slave_result =
if haskey(jld2_slavelogs, "$(row.psboard_id)_$(row.runid)_longrun.txt")
jld2_slavelogs["$(row.psboard_id)_$(row.runid)_longrun.txt"]
else
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
joinpath(
"../test/input/slavelogs/main/",
"$(row.psboard_id)_$(row.runid)_longrun.txt",
),
)
end
asdtp_results = slave_result.asdtp
@assert length(asdtp_results) == 100
# @info "" first(asdtp_results)
@ -383,12 +356,17 @@ asdtp_result_total_pp7_vs_ch = let
asdtp_result_total_pp7_vs_ch = zeros(Int64, 32)
for row in eachrow(df)
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
slave_result =
if haskey(jld2_slavelogs, "$(row.psboard_id)_$(row.runid)_longrun.txt")
jld2_slavelogs["$(row.psboard_id)_$(row.runid)_longrun.txt"]
else
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
joinpath(
"../test/input/slavelogs/main/",
"$(row.psboard_id)_$(row.runid)_longrun.txt",
),
)
end
asdtp_results = slave_result.asdtp
@assert length(asdtp_results) == 100
# @info "" first(asdtp_results)
@ -434,6 +412,7 @@ barplot(
# ╟─26954da1-0820-490d-aa3e-79fd3603d28a
# ╠═a36886af-ff32-4f4f-a20f-f83bc333b8fd
# ╠═df5087be-18de-4714-b6d8-4ab54271b1fb
# ╠═566d3472-bd2d-489d-8736-857cb7ccaa93
# ╠═56d8896a-40e3-46e3-aa28-82f89476743b
# ╠═08032250-5893-445c-8662-459e5636ba81
# ╠═013948ff-ccde-4222-862b-7ecda62fd7f1

2000
examples/reset_failed_though_reconfig_done.csv
View file

File diff suppressed because it is too large Load diff