mirror of
https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git
synced 2025-07-03 10:09:24 +09:00
Use slavelog cache in notebooks (only pllld_count)
This commit is contained in:
parent
d824f2c938
commit
2bab455089
4 changed files with 115 additions and 2126 deletions
|
|
@ -11,6 +11,7 @@ begin
|
|||
|
||||
using PSBoardDataBase
|
||||
using SQLite
|
||||
using JLD2
|
||||
using DataFrames
|
||||
using CSV
|
||||
using CairoMakie
|
||||
|
|
@ -40,6 +41,9 @@ db = SQLite.DB("../psboard_qaqc.db")
|
|||
# ╔═╡ df5087be-18de-4714-b6d8-4ab54271b1fb
|
||||
SQLite.tables(db)
|
||||
|
||||
# ╔═╡ 566d3472-bd2d-489d-8736-857cb7ccaa93
|
||||
jld2_slavelogs = jldopen("../slavelog.jld2")
|
||||
|
||||
# ╔═╡ 56d8896a-40e3-46e3-aa28-82f89476743b
|
||||
qaqc_runs = DBInterface.execute(db, "select * from qaqc_runs") |> DataFrame
|
||||
|
||||
|
|
@ -71,7 +75,11 @@ map(eachrow(qaqc_extra_run_campaign_6)) do row
|
|||
@info "file $(logfile) not found"
|
||||
return missing
|
||||
end
|
||||
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
|
||||
slave_result = if haskey(jld2_slavelogs, logfilename)
|
||||
jld2_slavelogs[logfilename]
|
||||
else
|
||||
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
|
||||
end
|
||||
map(slave_result.asdtp) do asdtp_result
|
||||
asdtp_result.pllld_fail_counter
|
||||
end
|
||||
|
|
@ -90,8 +98,11 @@ df_pllld_counts_camp6 = combine(
|
|||
@warn "logfile $(logfile) not found"
|
||||
return []
|
||||
end
|
||||
slave_result =
|
||||
slave_result = if haskey(jld2_slavelogs, logfilename)
|
||||
jld2_slavelogs[logfilename]
|
||||
else
|
||||
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
|
||||
end
|
||||
return map(slave_result.asdtp) do asdtp_result
|
||||
(
|
||||
psboard_id = sdf.psboard_id[1],
|
||||
|
|
@ -184,115 +195,66 @@ combine(
|
|||
)
|
||||
|
||||
# ╔═╡ a0baf37c-8b05-49c1-a6a4-027f3404f8b4
|
||||
df_reset_failed_though_reconfig_done = transform(
|
||||
qaqc_extra_run_results,
|
||||
[:psboard_id, :runid] =>
|
||||
ByRow(
|
||||
(psboard_id, runid) -> begin
|
||||
broken_list = @NamedTuple{psbid::Int64, runid::Int64}[
|
||||
(psbid = 1050, runid = 385),
|
||||
(psbid = 1113, runid = 392),
|
||||
(psbid = 1121, runid = 392),
|
||||
(psbid = 131, runid = 38),
|
||||
(psbid = 133, runid = 38),
|
||||
(psbid = 136, runid = 38),
|
||||
(psbid = 137, runid = 38),
|
||||
(psbid = 1545, runid = 474),
|
||||
(psbid = 16959, runid = 48),
|
||||
(psbid = 16959, runid = 59),
|
||||
(psbid = 198, runid = 251),
|
||||
(psbid = 202, runid = 45),
|
||||
(psbid = 203, runid = 45),
|
||||
(psbid = 205, runid = 45),
|
||||
(psbid = 206, runid = 45),
|
||||
(psbid = 206, runid = 57),
|
||||
(psbid = 206, runid = 59),
|
||||
(psbid = 208, runid = 45),
|
||||
(psbid = 210, runid = 45),
|
||||
(psbid = 214, runid = 45),
|
||||
(psbid = 215, runid = 48),
|
||||
(psbid = 221, runid = 55),
|
||||
(psbid = 223, runid = 48),
|
||||
(psbid = 224, runid = 48),
|
||||
(psbid = 229, runid = 48),
|
||||
(psbid = 231, runid = 48),
|
||||
(psbid = 234, runid = 48),
|
||||
(psbid = 235, runid = 48),
|
||||
(psbid = 237, runid = 45),
|
||||
(psbid = 238, runid = 45),
|
||||
(psbid = 239, runid = 45),
|
||||
(psbid = 263, runid = 69),
|
||||
(psbid = 329, runid = 69),
|
||||
(psbid = 339, runid = 69),
|
||||
(psbid = 42, runid = 178),
|
||||
(psbid = 488791279, runid = 63),
|
||||
(psbid = 48879, runid = 47),
|
||||
(psbid = 723, runid = 436),
|
||||
]
|
||||
if (; psbid = psboard_id, runid) in broken_list
|
||||
@error "broken file psbid: $(psboard_id), runid: $(runid)"
|
||||
return (;
|
||||
psboard_id,
|
||||
runid,
|
||||
reset_failed_though_reconfig_done = missing,
|
||||
)
|
||||
end
|
||||
@time df_reset_failed_though_reconfig_done = let
|
||||
broken_list = filter(:is_slavelog_valid => !=(1), qaqc_extra_run_results)
|
||||
transform(
|
||||
qaqc_extra_run_results,
|
||||
[:psboard_id, :runid] =>
|
||||
ByRow(
|
||||
(psboard_id, runid) -> begin
|
||||
if psboard_id in broken_list.psboard_id && runid in broken_list.runid
|
||||
return (;
|
||||
psboard_id,
|
||||
runid,
|
||||
reset_failed_though_reconfig_done = missing,
|
||||
)
|
||||
end
|
||||
|
||||
logfilename = "$(psboard_id)_$(runid)_longrun.txt"
|
||||
logfile = joinpath("../test/input/slavelogs/main/", logfilename)
|
||||
!ispath(logfile) && begin
|
||||
@warn "logfile $(logfile) not found"
|
||||
if runid == 99
|
||||
@info "skipping runid 99 since it is not exactly 100 run"
|
||||
return (
|
||||
psboard_id = psboard_id,
|
||||
runid = runid,
|
||||
reset_failed_though_reconfig_done = missing,
|
||||
)
|
||||
end
|
||||
|
||||
logfilename = "$(psboard_id)_$(runid)_longrun.txt"
|
||||
slave_result = if haskey(jld2_slavelogs, logfilename)
|
||||
jld2_slavelogs[logfilename]
|
||||
else
|
||||
logfile = joinpath("../test/input/slavelogs/main/", logfilename)
|
||||
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
|
||||
end
|
||||
return (
|
||||
psboard_id = psboard_id,
|
||||
runid = runid,
|
||||
reset_failed_though_reconfig_done = missing,
|
||||
reset_failed_though_reconfig_done = count(
|
||||
slave_result.asdtp,
|
||||
) do asdtp_result
|
||||
asdtp_result.reconfig_done_2 != asdtp_result.reconfig_done
|
||||
end,
|
||||
)
|
||||
end
|
||||
if runid == 99
|
||||
@info "skipping runid 99 since it is not exactly 100 run"
|
||||
return (
|
||||
psboard_id = psboard_id,
|
||||
runid = runid,
|
||||
reset_failed_though_reconfig_done = missing,
|
||||
)
|
||||
end
|
||||
if psboard_id == 1545 && runid == 474
|
||||
@info "skipping psbid 1545 runid 474 since it is broken"
|
||||
return (
|
||||
psboard_id = psboard_id,
|
||||
runid = runid,
|
||||
reset_failed_though_reconfig_done = missing,
|
||||
)
|
||||
end
|
||||
slave_result = try
|
||||
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(logfile)
|
||||
catch e
|
||||
@error "Failed to parse slavelog file: $(logfile)"
|
||||
throw(e)
|
||||
end
|
||||
return (
|
||||
psboard_id = psboard_id,
|
||||
runid = runid,
|
||||
reset_failed_though_reconfig_done = count(
|
||||
slave_result.asdtp,
|
||||
) do asdtp_result
|
||||
asdtp_result.reconfig_done_2 != asdtp_result.reconfig_done
|
||||
end,
|
||||
)
|
||||
end,
|
||||
) => [:psboard_id, :runid, :reset_failed_though_reconfig_done],
|
||||
)
|
||||
end,
|
||||
) => [:psboard_id, :runid, :reset_failed_though_reconfig_done],
|
||||
)
|
||||
end
|
||||
|
||||
# ╔═╡ 20633b30-a703-47bc-92a6-8743abbd375c
|
||||
sort(dropmissing(df_reset_failed_though_reconfig_done, :reset_failed_though_reconfig_done), :reset_failed_though_reconfig_done, rev = true)
|
||||
sort(
|
||||
dropmissing(df_reset_failed_though_reconfig_done, :reset_failed_though_reconfig_done),
|
||||
:reset_failed_though_reconfig_done,
|
||||
rev = true,
|
||||
)
|
||||
|
||||
# ╔═╡ 9cf8e7cb-a1f6-4c5b-8c3b-3872d2020e57
|
||||
result =
|
||||
PSBoardDataBase.SlaveLogParser.parse_slavelog_file("../test/input/slavelogs/main/1070_310_longrun.txt")
|
||||
result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
|
||||
"../test/input/slavelogs/main/1070_310_longrun.txt",
|
||||
)
|
||||
|
||||
# ╔═╡ 9b055b29-cf54-457d-8776-39430554ef9f
|
||||
findall(result.asdtp) do result
|
||||
result.reconfig_done != result.reconfig_done_2
|
||||
result.reconfig_done != result.reconfig_done_2
|
||||
end
|
||||
|
||||
# ╔═╡ d5d479b5-4d55-4a55-aa5f-893141981838
|
||||
|
|
@ -309,7 +271,13 @@ let
|
|||
end
|
||||
|
||||
# ╔═╡ dfeb3900-539f-4d8d-8103-edb67cf4e890
|
||||
CSV.write("reset_failed_though_reconfig_done.csv", df_reset_failed_though_reconfig_done[:, [:psboard_id, :runid, :reset_failed_though_reconfig_done]])
|
||||
# CSV.write(
|
||||
# "reset_failed_though_reconfig_done.csv",
|
||||
# df_reset_failed_though_reconfig_done[
|
||||
# :,
|
||||
# [:psboard_id, :runid, :reset_failed_though_reconfig_done],
|
||||
# ],
|
||||
# )
|
||||
|
||||
# ╔═╡ 332b7481-f6dc-4027-a5c5-cf58d5b6acd3
|
||||
md"""
|
||||
|
|
@ -325,12 +293,17 @@ asdtp_result_total_fail_vs_ppasic = let
|
|||
|
||||
asdtp_result_total_vs_ppasic = zeros(Int64, 8)
|
||||
for row in eachrow(df)
|
||||
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
|
||||
joinpath(
|
||||
"../test/input/slavelogs/main/",
|
||||
"$(row.psboard_id)_$(row.runid)_longrun.txt",
|
||||
),
|
||||
)
|
||||
slave_result =
|
||||
if haskey(jld2_slavelogs, "$(row.psboard_id)_$(row.runid)_longrun.txt")
|
||||
jld2_slavelogs["$(row.psboard_id)_$(row.runid)_longrun.txt"]
|
||||
else
|
||||
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
|
||||
joinpath(
|
||||
"../test/input/slavelogs/main/",
|
||||
"$(row.psboard_id)_$(row.runid)_longrun.txt",
|
||||
),
|
||||
)
|
||||
end
|
||||
asdtp_results = slave_result.asdtp
|
||||
@assert length(asdtp_results) == 100
|
||||
# @info "" first(asdtp_results)
|
||||
|
|
@ -383,12 +356,17 @@ asdtp_result_total_pp7_vs_ch = let
|
|||
|
||||
asdtp_result_total_pp7_vs_ch = zeros(Int64, 32)
|
||||
for row in eachrow(df)
|
||||
slave_result = PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
|
||||
joinpath(
|
||||
"../test/input/slavelogs/main/",
|
||||
"$(row.psboard_id)_$(row.runid)_longrun.txt",
|
||||
),
|
||||
)
|
||||
slave_result =
|
||||
if haskey(jld2_slavelogs, "$(row.psboard_id)_$(row.runid)_longrun.txt")
|
||||
jld2_slavelogs["$(row.psboard_id)_$(row.runid)_longrun.txt"]
|
||||
else
|
||||
PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
|
||||
joinpath(
|
||||
"../test/input/slavelogs/main/",
|
||||
"$(row.psboard_id)_$(row.runid)_longrun.txt",
|
||||
),
|
||||
)
|
||||
end
|
||||
asdtp_results = slave_result.asdtp
|
||||
@assert length(asdtp_results) == 100
|
||||
# @info "" first(asdtp_results)
|
||||
|
|
@ -434,6 +412,7 @@ barplot(
|
|||
# ╟─26954da1-0820-490d-aa3e-79fd3603d28a
|
||||
# ╠═a36886af-ff32-4f4f-a20f-f83bc333b8fd
|
||||
# ╠═df5087be-18de-4714-b6d8-4ab54271b1fb
|
||||
# ╠═566d3472-bd2d-489d-8736-857cb7ccaa93
|
||||
# ╠═56d8896a-40e3-46e3-aa28-82f89476743b
|
||||
# ╠═08032250-5893-445c-8662-459e5636ba81
|
||||
# ╠═013948ff-ccde-4222-862b-7ecda62fd7f1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue