mirror of
https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git
synced 2025-06-08 22:15:44 +09:00
673 lines
19 KiB
Julia
673 lines
19 KiB
Julia
"""
|
|
insert_qaqc_campaign_id(db::SQLite.DB)
|
|
|
|
Fill qaqc_campaigns table in `db`.
|
|
"""
|
|
function insert_qaqc_campaign_id(db::SQLite.DB)
|
|
campaigns = [1, 2, 3]
|
|
dates = [
|
|
(DateTime(2024, 7, 22), DateTime(2024, 7, 24)),
|
|
(DateTime(2024, 8, 6), DateTime(2024, 8, 9)),
|
|
(DateTime(2024, 9, 10), DateTime(2024, 9, 12)),
|
|
]
|
|
stmt = DBInterface.prepare(
|
|
db,
|
|
sql"INSERT INTO qaqc_campaigns VALUES (:id, :start_date, :end_date, :note)",
|
|
)
|
|
DBInterface.executemany(
|
|
stmt,
|
|
(
|
|
id = campaigns,
|
|
start_date = dates .|> (x -> x[1]) .|> string,
|
|
end_date = dates .|> (x -> x[2]) .|> string,
|
|
note = fill(nothing, size(campaigns)),
|
|
),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
insert_qaqc_positions(db::SQLite.DB)
|
|
|
|
Fill qaqc_positions table in `db`.
|
|
"""
|
|
function insert_qaqc_positions(db::SQLite.DB)
|
|
stmt = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_positions
|
|
VALUES(
|
|
:id,
|
|
:name,
|
|
:station,
|
|
:position
|
|
)
|
|
""",
|
|
)
|
|
DBInterface.executemany(
|
|
stmt,
|
|
(
|
|
id = 1:18,
|
|
name = ["B-$i-$j" for i in 0:1 for j in 1:9],
|
|
station = [fill(0, 9); fill(1, 9)],
|
|
position = [collect(1:9); collect(1:9)],
|
|
),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
prepare_single_result_df(single_result_table::DataFrame)
|
|
|
|
Common preprocess(format) function for single result table.
|
|
|
|
# Detail
|
|
- convert `timestamp` to [`DateTime`](@extref Dates.DateTime)
|
|
"""
|
|
function prepare_single_result_df(single_result_table::DataFrame)
|
|
df = copy(single_result_table, copycols = true)
|
|
|
|
# timestamp format: 2024-08-07T06:18:09Z
|
|
# ignore the last 'Z' => [1:end-1]
|
|
transform!(
|
|
df,
|
|
:timestamp =>
|
|
ByRow(s -> ismissing(s) ? missing : DateTime(s[1:(end - 1)])) => :timestamp,
|
|
)
|
|
|
|
return df
|
|
end
|
|
|
|
"""
|
|
prepare_runlist_df(runlist_table::DataFrame)
|
|
|
|
Common preprocess(format) function for runlist table.
|
|
"""
|
|
function prepare_runlist_df(runlist_table::DataFrame)
|
|
df = copy(runlist_table, copycols = true)
|
|
end
|
|
|
|
"""
|
|
add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame)
|
|
|
|
Add PS Board IDs from single test result table.
|
|
Assume that all PS Boards are included in `single_result_table`.
|
|
"""
|
|
function add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame)
|
|
df = combine(groupby(single_result_table, :motherboard_id)) do df
|
|
if df.daughterboard |> unique |> length |> ==(1)
|
|
return (daughterboard = df.daughterboard |> first,)
|
|
end
|
|
df = sort(df, :timestamp)
|
|
dropmissing!(df, :daughterboard)
|
|
return (daughterboard = df.daughterboard |> last,)
|
|
end
|
|
filter!(:motherboard_id => !=(999999), df)
|
|
|
|
stmt = DBInterface.prepare(
|
|
db,
|
|
sql"INSERT INTO ps_boards VALUES (:psbid, :daughterboardid)",
|
|
)
|
|
DBInterface.executemany(
|
|
stmt,
|
|
(psbid = df.motherboard_id, daughterboardid = df.daughterboard),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
add_qaqc_runlist_from_runlist(db::SQLite.DB, runlist_table::DataFrame)
|
|
|
|
Add QAQC runs to `qaqc_runs` table in `db` from RUNLIST csv.
|
|
"""
|
|
function add_qaqc_runlist_from_runlist(db::SQLite.DB, runlist_table::DataFrame)
|
|
stmt_insert_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_runs
|
|
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
|
""",
|
|
)
|
|
runlist_table = dropmissing(runlist_table, Symbol("Run ID"))
|
|
@assert allunique(runlist_table, Symbol("Run ID"))
|
|
for row in eachrow(runlist_table)
|
|
try
|
|
DBInterface.execute(
|
|
stmt_insert_runid,
|
|
(
|
|
runid = row.var"Run ID",
|
|
campaign_id = row.var"Campaign ID",
|
|
run_datetime = nothing,
|
|
note = row.comment,
|
|
shifter = row.var"Shifter name",
|
|
logfile = nothing,
|
|
shiftscript_ver = nothing,
|
|
),
|
|
)
|
|
catch e
|
|
@error "error in putting run list" e
|
|
@info "row" row
|
|
end
|
|
end
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
get_campaign_id_from_run_id(runid::Integer)
|
|
"""
|
|
function get_campaign_id_from_run_id(runid::Integer)
|
|
if runid < 63
|
|
1
|
|
elseif runid < 98
|
|
2
|
|
elseif runid < 169
|
|
3
|
|
else
|
|
@error "Fix this function"
|
|
DomainError("runid $(runid) is not registered to the software")
|
|
end
|
|
end
|
|
|
|
"""
|
|
add_qaqc_single_result(
|
|
db::SQLite.DB,
|
|
single_result_table::DataFrame,
|
|
runlist_table::DataFrame,
|
|
) -> nothing
|
|
|
|
Fill `qaqc_single_run_results` in `db` from single result table DataFrame.
|
|
Additionaly, it
|
|
1. automatically add `runid` if it's not in `qaqc_runs` table in `db`.
|
|
2. automatically update fields in `qaqc_runs` table.
|
|
"""
|
|
function add_qaqc_single_result(
|
|
db::SQLite.DB,
|
|
single_result_table::DataFrame,
|
|
runlist_table::DataFrame,
|
|
)
|
|
single_result_table = prepare_single_result_df(single_result_table)
|
|
|
|
position_id_map =
|
|
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
|
(i, s) = x
|
|
s => i
|
|
end) |> Dict
|
|
|
|
stmt_search_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
SELECT id
|
|
FROM qaqc_runs
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_insert_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_runs
|
|
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
|
""",
|
|
)
|
|
stmt_update_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
UPDATE qaqc_runs
|
|
SET run_datetime = :run_datetime, shifter = :shifter, logfile = :logfile, shiftscript_ver = :shiftscript_ver
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_insert_result = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO
|
|
qaqc_single_run_results(
|
|
runid,
|
|
psboard_id,
|
|
daughterboard_id,
|
|
position,
|
|
resistance_test_passed,
|
|
qspip,
|
|
recov,
|
|
power,
|
|
clock,
|
|
asdtp,
|
|
reset,
|
|
qaqc_result,
|
|
note
|
|
)
|
|
VALUES (
|
|
:runid,
|
|
:psboard_id,
|
|
:daughterboard_id,
|
|
:position,
|
|
:resistance_test_passed,
|
|
:qspip,
|
|
:recov,
|
|
:power,
|
|
:clock,
|
|
:asdtp,
|
|
:reset,
|
|
:qaqc_result,
|
|
:note
|
|
)
|
|
""",
|
|
)
|
|
stmt_insert_resistance = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_resistance_check(psb_id, passed)
|
|
VALUES (:psboard_id, :ispassed)
|
|
""",
|
|
)
|
|
|
|
for row in eachrow(single_result_table)
|
|
if ismissing(row.runid)
|
|
@assert contains("resistance")(row.comment) || contains("CN15")(row.comment) "Unexpected row with id $(row.motherboard_id) $(row.comment)"
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_resistance,
|
|
(psboard_id = row.motherboard_id, ispassed = false),
|
|
)
|
|
|
|
continue
|
|
end
|
|
|
|
# Add run if it's not in `qaqc_runs` table
|
|
# or update info on the run (such as datetime)
|
|
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
|
|
campaign_id = get_campaign_id_from_run_id(row.runid)
|
|
comment = let
|
|
row_run = filter(
|
|
Symbol("Run ID") => x -> !ismissing(x) && x == row.runid,
|
|
runlist_table,
|
|
)
|
|
if !isempty(row_run)
|
|
row_run.comment
|
|
else
|
|
""
|
|
end
|
|
end
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_runid,
|
|
(
|
|
runid = row.runid,
|
|
campaign_id = campaign_id,
|
|
run_datetime = row.timestamp |> string,
|
|
note = comment,
|
|
shifter = row.shifter,
|
|
logfile = row.qaqc_log_file,
|
|
shiftscript_ver = row.shiftscript_ver,
|
|
),
|
|
)
|
|
else
|
|
DBInterface.execute(
|
|
stmt_update_runid,
|
|
(
|
|
runid = row.runid,
|
|
run_datetime = row.timestamp |> string,
|
|
shifter = row.shifter,
|
|
logfile = row.qaqc_log_file,
|
|
shiftscript_ver = row.shiftscript_ver,
|
|
),
|
|
)
|
|
end
|
|
|
|
# resistance
|
|
DBInterface.execute(
|
|
stmt_insert_resistance,
|
|
(psboard_id = row.motherboard_id, ispassed = true),
|
|
)
|
|
|
|
# main result
|
|
DBInterface.execute(
|
|
stmt_insert_result,
|
|
(
|
|
runid = row.runid,
|
|
psboard_id = row.motherboard_id,
|
|
daughterboard_id = row.daughterboard,
|
|
position = position_id_map[row.position],
|
|
resistance_test_passed = true,
|
|
qspip = row.qspip,
|
|
recov = row.recov,
|
|
power = row.power,
|
|
clock = row.clock,
|
|
asdtp = row.asdtp,
|
|
reset = row.reset,
|
|
qaqc_result = row.qaqc_result,
|
|
note = row.comment,
|
|
),
|
|
)
|
|
end
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame
|
|
|
|
Format `qaqc_dispatch` DataFrame from exported CSV.
|
|
Used in [`add_qaqc_dispatch`](@ref).
|
|
"""
|
|
function prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame
|
|
df = copy(raw_dispatch_table, copycols = true)
|
|
transform!(
|
|
df,
|
|
[Symbol("Column$i") for i in 2:ncol(df)] => ByRow((s...) -> join(s |> skipmissing)) => :comment,
|
|
)
|
|
select!(df, [1, ncol(df)])
|
|
rename!(df, [:psboard_id, :comment])
|
|
|
|
transform!(
|
|
df,
|
|
:psboard_id => (vs -> accumulate(vs; init = 0) do x, y
|
|
ispsbid = startswith("PS")
|
|
if ispsbid(y)
|
|
x
|
|
else
|
|
match(r"(\d+)", y) |> first
|
|
end
|
|
end) => :campaign_id,
|
|
)
|
|
transform!(
|
|
df,
|
|
:psboard_id =>
|
|
ByRow(s -> startswith("PS")(s) ? parse(Int64, s[3:end]) : missing) =>
|
|
:psboard_id,
|
|
)
|
|
dropmissing!(df, :psboard_id)
|
|
|
|
df
|
|
end
|
|
|
|
"""
|
|
add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
|
|
|
Fill `qaqc_dispatch` table in `db` from `dispatch_table`.
|
|
"""
|
|
function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
|
dispatch_table = prepare_dispatch_table(dispatch_table)
|
|
|
|
# TODO: provide datetime
|
|
stmt = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_dispatch(qaqc_campaign_id, psb_id, source_place, destination, time)
|
|
VALUES (:campaign_id, :psboard_id, "KEK", "GND", NULL)
|
|
""",
|
|
)
|
|
|
|
DBInterface.executemany(
|
|
stmt,
|
|
(campaign_id = dispatch_table.campaign_id, psboard_id = dispatch_table.psboard_id),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString) -> nothing
|
|
|
|
Add qaqc run list from master log files in `logs_dir`.
|
|
Currently, it adds long runs and run with id 20-23 only (since normal runs are added from single run results table).
|
|
"""
|
|
function add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString)
|
|
stmt_search_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
SELECT id, run_datetime
|
|
FROM qaqc_runs
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_update_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
UPDATE qaqc_runs
|
|
SET run_datetime = :run_datetime, shifter = :shifter, logfile = :logfile, shiftscript_ver = :shiftscript_ver
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_insert_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_runs
|
|
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
|
""",
|
|
)
|
|
|
|
is_run_to_add(log_file) = begin
|
|
m = match(r"(\d+)\.log", log_file)
|
|
contains("_long.log")(log_file) ||
|
|
!isnothing(m) && (
|
|
begin
|
|
num = parse(Int64, m[1])
|
|
20 <= num <= 23 || num == 27 || num == 28
|
|
end
|
|
)
|
|
end
|
|
|
|
longrun_logs = readdir(logs_dir; join = true) |> filter(is_run_to_add)
|
|
# longrun_logs = readdir(logs_dir; join = true) |> filter(contains("_long.log"))
|
|
for longrun_log in longrun_logs
|
|
run_metadata = QaqcMasterLog.parse_master_log(longrun_log)
|
|
if isnothing(run_metadata)
|
|
continue
|
|
end
|
|
current_rundb =
|
|
DBInterface.execute(stmt_search_runid, (; runid = run_metadata.runid)) |>
|
|
DataFrame
|
|
if !isempty(current_rundb.id)
|
|
# runid is already in the database
|
|
if any(ismissing, current_rundb.run_datetime)
|
|
# add timestamp, logfile, ...etc
|
|
DBInterface.execute(
|
|
stmt_update_runid,
|
|
(
|
|
runid = run_metadata.runid,
|
|
run_datetime = run_metadata.timestamp |> string,
|
|
shifter = run_metadata.shifters,
|
|
logfile = splitdir(longrun_log) |> last,
|
|
shiftscript_ver = run_metadata.shiftscript_version,
|
|
),
|
|
)
|
|
end
|
|
continue
|
|
end
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_runid,
|
|
(
|
|
runid = run_metadata.runid,
|
|
campaign_id = get_campaign_id_from_run_id(run_metadata.runid),
|
|
run_datetime = run_metadata.timestamp |> string,
|
|
note = "",
|
|
shifter = run_metadata.shifters,
|
|
logfile = splitdir(longrun_log) |> last,
|
|
shiftscript_ver = run_metadata.shiftscript_version,
|
|
),
|
|
)
|
|
end
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
prepare_100test_table(table::DataFrame)::DataFrame
|
|
|
|
Format 100test result `table` from exported CSV.
|
|
Used in [`add_qaqc_100test_result`](@ref).
|
|
|
|
# Detail
|
|
- Format `motherboard ID`s
|
|
- `PS00xxxx` -> `Int64(xxxx)`
|
|
- `xxxx` -> `Int64(xxxx)`
|
|
- For `psbid == 484` and `runid == 115` results, make all result fields to `missing` since they contain abnormal strings.
|
|
"""
|
|
function prepare_100test_table(table::DataFrame)::DataFrame
|
|
df = copy(table, copycols = true)
|
|
|
|
transform!(
|
|
df,
|
|
Symbol("motherboard ID") =>
|
|
ByRow(s -> if startswith("PS")(s)
|
|
parse(Int64, s[3:end])
|
|
else
|
|
parse(Int64, s)
|
|
end) => :motherboard_id,
|
|
)
|
|
transform!(
|
|
df,
|
|
Cols(:motherboard_id, :runid, 4:13) =>
|
|
ByRow((psbid, runid, items...) -> if psbid == 484 && runid == 115
|
|
missings(items |> length)
|
|
else
|
|
items
|
|
end) => names(df)[4:13],
|
|
)
|
|
|
|
df
|
|
end
|
|
|
|
"""
|
|
get_num_tests_for_extra_runs(runid::Int64)
|
|
|
|
Get number of tests for extra QAQC runs.
|
|
They are usually 100.
|
|
|
|
Current abnormal runs:
|
|
|
|
| runid | # of runs |
|
|
|-------|-----------|
|
|
| 99| 246|
|
|
"""
|
|
function get_num_tests_for_extra_runs(runid::Int64)
|
|
if runid == 99
|
|
246
|
|
else
|
|
100
|
|
end
|
|
end
|
|
|
|
"""
|
|
add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) -> nothing
|
|
|
|
Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
|
|
which is converted from a raw exported CSV.
|
|
|
|
# Detail
|
|
- skips psboards in `resistance_test_passed` with `passed == false`
|
|
"""
|
|
function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
|
|
position_id_map =
|
|
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
|
(i, s) = x
|
|
s => i
|
|
end) |> Dict
|
|
|
|
table = prepare_100test_table(table)
|
|
|
|
stmt_search_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
SELECT id
|
|
FROM qaqc_runs
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_search_resistance_error = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
SELECT psb_id
|
|
FROM qaqc_resistance_check
|
|
WHERE psb_id = :psboard_id AND passed = 0
|
|
""",
|
|
)
|
|
stmt_insert_result = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO
|
|
qaqc_extra_run_results (
|
|
runid,
|
|
psboard_id,
|
|
position,
|
|
num_tests,
|
|
insufficient_reset_with_10,
|
|
reset_failed_though_reconfig_done,
|
|
always_hit_flag_true,
|
|
dac_is_0,
|
|
bcid_shift,
|
|
efficiency_99percent,
|
|
bcid_fail_111,
|
|
bcid_fail_000,
|
|
low_efficiency,
|
|
bcid_fail,
|
|
invalid_register_value,
|
|
power_out_of_range,
|
|
note
|
|
)
|
|
VALUES (
|
|
:runid,
|
|
:psboard_id,
|
|
:position,
|
|
:num_tests,
|
|
:insufficient_reset_with_10,
|
|
:reset_failed_though_reconfig_done,
|
|
:always_hit_flag_true,
|
|
:dac_is_0,
|
|
:bcid_shift,
|
|
:efficiency_99percent,
|
|
:bcid_fail_111,
|
|
:bcid_fail_000,
|
|
:low_efficiency,
|
|
:bcid_fail,
|
|
:invalid_register_value,
|
|
:power_out_of_range,
|
|
:note
|
|
)
|
|
""",
|
|
)
|
|
|
|
for row in eachrow(table)
|
|
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
|
|
# search for resistance error
|
|
if !isempty(
|
|
DBInterface.execute(
|
|
stmt_search_resistance_error,
|
|
(; psboard_id = row.motherboard_id),
|
|
),
|
|
)
|
|
continue
|
|
end
|
|
error("Runid $(row.runid) not found in `qaqc_runs` table.")
|
|
end
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_result,
|
|
(
|
|
runid = row.runid,
|
|
psboard_id = row.motherboard_id,
|
|
position = position_id_map[row.position],
|
|
num_tests = get_num_tests_for_extra_runs(row.runid),
|
|
insufficient_reset_with_10 = row.var"10回reset足りず",
|
|
reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない",
|
|
always_hit_flag_true = row.var"always_hit_flag",
|
|
dac_is_0 = row.var"DAC = 0",
|
|
bcid_shift = row.var"DAC = 0",
|
|
efficiency_99percent = row.var"efficiency 99%",
|
|
bcid_fail_111 = row.var"BCID 0:0:0",
|
|
bcid_fail_000 = row.var"BCID 1:1:1",
|
|
low_efficiency = row.var"low efficiency",
|
|
bcid_fail = row.var"BCID fail",
|
|
invalid_register_value = row.var"invalid register values",
|
|
power_out_of_range = row.var"power out of range",
|
|
note = row.Column20,
|
|
),
|
|
)
|
|
end
|
|
|
|
nothing
|
|
end
|