mirror of
https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git
synced 2025-06-11 07:19:23 +09:00
add: import 100 run results + more
- master log parser to get run info - test for master log parser - master log reader - note field for single run - fix datetime manipulation for single result table
This commit is contained in:
parent
f5e6caf936
commit
21f38e83db
9 changed files with 1505 additions and 16 deletions
|
|
@ -130,11 +130,23 @@ function add_qaqc_runlist(db::SQLite.DB, runlist_table::DataFrame)
|
|||
nothing
|
||||
end
|
||||
|
||||
function get_campaign_id_from_run_id(runid::Integer)
|
||||
if runid < 63
|
||||
1
|
||||
elseif runid < 98
|
||||
2
|
||||
else
|
||||
3
|
||||
end
|
||||
end
|
||||
|
||||
function add_qaqc_single_result(
|
||||
db::SQLite.DB,
|
||||
single_result_table::DataFrame,
|
||||
runlist_table::DataFrame,
|
||||
)
|
||||
single_result_table = prepare_single_result_df(single_result_table)
|
||||
|
||||
position_id_map =
|
||||
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
||||
(i, s) = x
|
||||
|
|
@ -180,7 +192,8 @@ function add_qaqc_single_result(
|
|||
clock,
|
||||
asdtp,
|
||||
reset,
|
||||
qaqc_result
|
||||
qaqc_result,
|
||||
note
|
||||
)
|
||||
VALUES (
|
||||
:runid,
|
||||
|
|
@ -194,7 +207,8 @@ function add_qaqc_single_result(
|
|||
:clock,
|
||||
:asdtp,
|
||||
:reset,
|
||||
:qaqc_result
|
||||
:qaqc_result,
|
||||
:note
|
||||
)
|
||||
""",
|
||||
)
|
||||
|
|
@ -221,13 +235,7 @@ function add_qaqc_single_result(
|
|||
# Add run if it's not in `qaqc_runs` table
|
||||
# or update info on the run (such as datetime)
|
||||
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
|
||||
campaign_id = if row.runid < 63
|
||||
1
|
||||
elseif row.runid < 98
|
||||
2
|
||||
else
|
||||
3
|
||||
end
|
||||
campaign_id = get_campaign_id_from_run_id(row.runid)
|
||||
comment = let
|
||||
row_run = filter(
|
||||
Symbol("Run ID") => x -> !ismissing(x) && x == row.runid,
|
||||
|
|
@ -245,7 +253,7 @@ function add_qaqc_single_result(
|
|||
(
|
||||
runid = row.runid,
|
||||
campaign_id = campaign_id,
|
||||
run_datetime = row.timestamp,
|
||||
run_datetime = row.timestamp |> string,
|
||||
note = comment,
|
||||
shifter = row.shifter,
|
||||
logfile = row.qaqc_log_file,
|
||||
|
|
@ -257,7 +265,7 @@ function add_qaqc_single_result(
|
|||
stmt_update_runid,
|
||||
(
|
||||
runid = row.runid,
|
||||
run_datetime = row.timestamp,
|
||||
run_datetime = row.timestamp |> string,
|
||||
shifter = row.shifter,
|
||||
logfile = row.qaqc_log_file,
|
||||
shiftscript_ver = row.shiftscript_ver,
|
||||
|
|
@ -287,6 +295,7 @@ function add_qaqc_single_result(
|
|||
asdtp = row.asdtp,
|
||||
reset = row.reset,
|
||||
qaqc_result = row.qaqc_result,
|
||||
note = row.comment,
|
||||
),
|
||||
)
|
||||
end
|
||||
|
|
@ -325,7 +334,7 @@ function prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame
|
|||
df
|
||||
end
|
||||
|
||||
function add_qaqcdispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
||||
function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
||||
dispatch_table = prepare_dispatch_table(dispatch_table)
|
||||
|
||||
# TODO: provide datetime
|
||||
|
|
@ -344,3 +353,197 @@ function add_qaqcdispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
|||
|
||||
nothing
|
||||
end
|
||||
|
||||
"""
|
||||
Add qaqc run list from master log files in `logs_dir`.
|
||||
Currently, it adds long runs and run with id 20-23 only (since normal runs are added from single run results table).
|
||||
"""
|
||||
function add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString)
|
||||
stmt_search_runid = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
SELECT id
|
||||
FROM qaqc_runs
|
||||
WHERE id = :runid
|
||||
""",
|
||||
)
|
||||
stmt_insert_runid = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
INSERT INTO qaqc_runs
|
||||
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
||||
""",
|
||||
)
|
||||
|
||||
is_run_to_add(log_file) = begin
|
||||
m = match(r"(\d+)\.log", log_file)
|
||||
contains("_long.log")(log_file) ||
|
||||
!isnothing(m) && (
|
||||
begin
|
||||
num = parse(Int64, m[1])
|
||||
20 <= num <= 23 || num == 27 || num == 28
|
||||
end
|
||||
)
|
||||
end
|
||||
|
||||
longrun_logs = readdir(logs_dir; join = true) |> filter(is_run_to_add)
|
||||
# longrun_logs = readdir(logs_dir; join = true) |> filter(contains("_long.log"))
|
||||
for longrun_log in longrun_logs
|
||||
run_metadata = QaqcMasterLog.parse_master_log(longrun_log)
|
||||
if isnothing(run_metadata)
|
||||
continue
|
||||
end
|
||||
|
||||
if !isempty(DBInterface.execute(stmt_search_runid, (; runid = run_metadata.runid)))
|
||||
# runid is already in the database
|
||||
continue
|
||||
end
|
||||
|
||||
DBInterface.execute(
|
||||
stmt_insert_runid,
|
||||
(
|
||||
runid = run_metadata.runid,
|
||||
campaign_id = get_campaign_id_from_run_id(run_metadata.runid),
|
||||
run_datetime = run_metadata.timestamp,
|
||||
note = "",
|
||||
shifter = run_metadata.shifters,
|
||||
logfile = splitdir(longrun_log) |> last,
|
||||
shiftscript_ver = run_metadata.shiftscript_version,
|
||||
),
|
||||
)
|
||||
end
|
||||
|
||||
nothing
|
||||
end
|
||||
|
||||
function prepare_100test_table(table::DataFrame)::DataFrame
|
||||
df = copy(table, copycols = true)
|
||||
|
||||
transform!(
|
||||
df,
|
||||
Symbol("motherboard ID") =>
|
||||
ByRow(s -> if startswith("PS")(s)
|
||||
parse(Int64, s[3:end])
|
||||
else
|
||||
parse(Int64, s)
|
||||
end) => :motherboard_id,
|
||||
)
|
||||
transform!(
|
||||
df,
|
||||
Cols(:motherboard_id, :runid, 4:13) =>
|
||||
ByRow((psbid, runid, items...) -> if psbid == 484 && runid == 115
|
||||
missings(items |> length)
|
||||
else
|
||||
items
|
||||
end) => names(df)[4:13],
|
||||
)
|
||||
|
||||
df
|
||||
end
|
||||
|
||||
function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
|
||||
position_id_map =
|
||||
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
||||
(i, s) = x
|
||||
s => i
|
||||
end) |> Dict
|
||||
|
||||
table = prepare_100test_table(table)
|
||||
|
||||
stmt_search_runid = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
SELECT id
|
||||
FROM qaqc_runs
|
||||
WHERE id = :runid
|
||||
""",
|
||||
)
|
||||
stmt_search_resistance_error = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
SELECT psb_id
|
||||
FROM qaqc_resistance_check
|
||||
WHERE psb_id = :psboard_id AND passed = 0
|
||||
""",
|
||||
)
|
||||
stmt_insert_result = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
INSERT INTO
|
||||
qaqc_extra_run_results (
|
||||
runid,
|
||||
psboard_id,
|
||||
position,
|
||||
num_tests,
|
||||
insufficient_reset_with_10,
|
||||
reset_failed_though_reconfig_done,
|
||||
always_hit_flag_true,
|
||||
dac_is_0,
|
||||
bcid_shift,
|
||||
efficiency_99percent,
|
||||
bcid_fail_111,
|
||||
bcid_fail_000,
|
||||
low_efficiency,
|
||||
bcid_fail,
|
||||
invalid_register_value,
|
||||
power_out_of_range,
|
||||
note
|
||||
)
|
||||
VALUES (
|
||||
:runid,
|
||||
:psboard_id,
|
||||
:position,
|
||||
100,
|
||||
:insufficient_reset_with_10,
|
||||
:reset_failed_though_reconfig_done,
|
||||
:always_hit_flag_true,
|
||||
:dac_is_0,
|
||||
:bcid_shift,
|
||||
:efficiency_99percent,
|
||||
:bcid_fail_111,
|
||||
:bcid_fail_000,
|
||||
:low_efficiency,
|
||||
:bcid_fail,
|
||||
:invalid_register_value,
|
||||
:power_out_of_range,
|
||||
:note
|
||||
)
|
||||
""",
|
||||
)
|
||||
|
||||
for row in eachrow(table)
|
||||
# TODO: get runid from master log file
|
||||
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
|
||||
# search for resistance error
|
||||
if !isempty(
|
||||
DBInterface.execute(stmt_search_resistance_error, (; psboard_id = row.motherboard_id)),
|
||||
)
|
||||
continue
|
||||
end
|
||||
end
|
||||
|
||||
DBInterface.execute(
|
||||
stmt_insert_result,
|
||||
(
|
||||
runid = row.runid,
|
||||
psboard_id = row.motherboard_id,
|
||||
position = position_id_map[row.position],
|
||||
insufficient_reset_with_10 = row.var"10回reset足りず",
|
||||
reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない",
|
||||
always_hit_flag_true = row.var"always_hit_flag",
|
||||
dac_is_0 = row.var"DAC = 0",
|
||||
bcid_shift = row.var"DAC = 0",
|
||||
efficiency_99percent = row.var"efficiency 99%",
|
||||
bcid_fail_111 = row.var"BCID 0:0:0",
|
||||
bcid_fail_000 = row.var"BCID 1:1:1",
|
||||
low_efficiency = row.var"low efficiency",
|
||||
bcid_fail = row.var"BCID fail",
|
||||
invalid_register_value = row.var"invalid register values",
|
||||
power_out_of_range = row.var"power out of range",
|
||||
note = row.Column20,
|
||||
),
|
||||
)
|
||||
end
|
||||
|
||||
nothing
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue