mirror of
https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git
synced 2025-06-08 05:55:42 +09:00
964 lines
29 KiB
Julia
964 lines
29 KiB
Julia
"""
|
|
insert_version_info(db::SQLite.DB)
|
|
|
|
Insert version information of this software as string.
|
|
"""
|
|
function insert_version_info(db::SQLite.DB)
|
|
stmt_insert_version = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO versions VALUES (:converter)
|
|
""",
|
|
)
|
|
@info "converter version info" pkgversion(@__MODULE__) |> string
|
|
DBInterface.execute(
|
|
stmt_insert_version,
|
|
(; converter = pkgversion(@__MODULE__) |> string),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
insert_qaqc_campaign_id(db::SQLite.DB)
|
|
|
|
Fill qaqc_campaigns table in `db`.
|
|
"""
|
|
function insert_qaqc_campaign_id(db::SQLite.DB)
|
|
campaigns = [1, 2, 3, 4, 5, 6, 7]
|
|
dates = [
|
|
(DateTime(2024, 7, 22), DateTime(2024, 7, 24)),
|
|
(DateTime(2024, 8, 6), DateTime(2024, 8, 9)),
|
|
(DateTime(2024, 9, 10), DateTime(2024, 9, 12)),
|
|
(DateTime(2024, 9, 30), DateTime(2024, 10, 4)),
|
|
(DateTime(2024, 11, 11), DateTime(2024, 11, 14)),
|
|
(DateTime(2024, 12, 9), DateTime(2024, 12, 12)),
|
|
(DateTime(2025, 1, 20), DateTime(2024, 1, 23)),
|
|
]
|
|
stmt_insert_campaigns = DBInterface.prepare(
|
|
db,
|
|
sql"INSERT INTO qaqc_campaigns VALUES (:id, :start_date, :end_date, :note)",
|
|
)
|
|
DBInterface.executemany(
|
|
stmt_insert_campaigns,
|
|
(
|
|
id = campaigns,
|
|
start_date = dates .|> (x -> x[1]) .|> string,
|
|
end_date = dates .|> (x -> x[2]) .|> string,
|
|
note = fill(nothing, size(campaigns)),
|
|
),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame)
|
|
|
|
Fill qaqc_positions table in `db`.
|
|
Argument `jathub_db_table` is for skew for each positions.
|
|
"""
|
|
function insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame)
|
|
dropmissing!(jathub_db_table, :psb_position)
|
|
transform!(
|
|
jathub_db_table,
|
|
Symbol("立ち上がり [ns]") => ByRow(Float64) => Symbol("立ち上がり [ns]"),
|
|
)
|
|
|
|
stmt_insert_positions = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_positions
|
|
VALUES(
|
|
:id,
|
|
:name,
|
|
:station,
|
|
:position,
|
|
:rising_ns
|
|
)
|
|
""",
|
|
)
|
|
DBInterface.executemany(
|
|
stmt_insert_positions,
|
|
(
|
|
id = 1:18,
|
|
name = ["B-$i-$j" for i in 0:1 for j in 1:9],
|
|
station = [fill(0, 9); fill(1, 9)],
|
|
position = [collect(1:9); collect(1:9)],
|
|
rising_ns = [
|
|
filter(
|
|
:psb_position => (s -> !ismissing(s) && s == "B-$i-$j"),
|
|
jathub_db_table,
|
|
).var"立ち上がり [ns]" |> first for i in 0:1 for j in 1:9
|
|
],
|
|
),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
prepare_single_result_df(single_result_table::DataFrame)
|
|
|
|
Common preprocess(format) function for single result table.
|
|
|
|
# Detail
|
|
- convert `timestamp` to [`DateTime`](@extref Dates.DateTime)
|
|
"""
|
|
function prepare_single_result_df(single_result_table::DataFrame)
|
|
df = copy(single_result_table, copycols = true)
|
|
|
|
# timestamp format: 2024-08-07T06:18:09Z
|
|
# ignore the last 'Z' => [1:end-1]
|
|
transform!(
|
|
df,
|
|
:timestamp =>
|
|
ByRow(s -> ismissing(s) ? missing : DateTime(s[1:(end - 1)])) => :timestamp,
|
|
)
|
|
|
|
return df
|
|
end
|
|
|
|
"""
|
|
prepare_runlist_df(runlist_table::DataFrame)
|
|
|
|
Common preprocess(format) function for runlist table.
|
|
"""
|
|
function prepare_runlist_df(runlist_table::DataFrame)
|
|
df = copy(runlist_table, copycols = true)
|
|
end
|
|
|
|
"""
|
|
add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame)
|
|
|
|
Add PS Board IDs from single test result table.
|
|
Assume that all PS Boards are included in `single_result_table`.
|
|
"""
|
|
function add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame)
|
|
df = combine(groupby(single_result_table, :motherboard_id)) do df
|
|
if df.daughterboard |> unique |> length |> ==(1)
|
|
return (daughterboard = df.daughterboard |> first,)
|
|
end
|
|
df = sort(df, :timestamp)
|
|
dropmissing!(df, :daughterboard)
|
|
return (daughterboard = df.daughterboard |> last,)
|
|
end
|
|
try
|
|
filter!(:motherboard_id => !=(999999), df)
|
|
catch
|
|
id_missings = findall(ismissing, df.motherboard_id)
|
|
if !isempty(id_missings)
|
|
@error "motherboard id at row $(id_missings) are missing. motherboard id cannot be null"
|
|
error("single_result_table format error")
|
|
end
|
|
end
|
|
|
|
stmt_insert_psbid = DBInterface.prepare(
|
|
db,
|
|
sql"INSERT INTO ps_boards VALUES (:psbid, :daughterboardid)",
|
|
)
|
|
DBInterface.executemany(
|
|
stmt_insert_psbid,
|
|
(psbid = df.motherboard_id, daughterboardid = df.daughterboard),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
add_qaqc_runlist_from_runlist(db::SQLite.DB, runlist_table::DataFrame)
|
|
|
|
Add QAQC runs to `qaqc_runs` table in `db` from RUNLIST csv.
|
|
"""
|
|
function add_qaqc_runlist_from_runlist(db::SQLite.DB, runlist_table::DataFrame)
|
|
stmt_insert_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_runs
|
|
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
|
""",
|
|
)
|
|
runlist_table = dropmissing(runlist_table, Symbol("Run ID"))
|
|
@assert allunique(runlist_table, Symbol("Run ID"))
|
|
for row in eachrow(runlist_table)
|
|
try
|
|
DBInterface.execute(
|
|
stmt_insert_runid,
|
|
(
|
|
runid = row.var"Run ID",
|
|
campaign_id = row.var"Campaign ID",
|
|
run_datetime = nothing,
|
|
note = row.comment,
|
|
shifter = row.var"Shifter name",
|
|
logfile = nothing,
|
|
shiftscript_ver = nothing,
|
|
),
|
|
)
|
|
catch e
|
|
@error "error in putting run list" e
|
|
@info "row" row
|
|
end
|
|
end
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
get_campaign_id_from_run_id(runid::Integer)
|
|
"""
|
|
function get_campaign_id_from_run_id(runid::Integer)
|
|
if runid < 63
|
|
1
|
|
elseif runid < 98
|
|
2
|
|
elseif runid < 188
|
|
3
|
|
elseif runid < 293
|
|
4
|
|
elseif runid < 354
|
|
5
|
|
elseif runid < 425
|
|
6
|
|
elseif 448 ≤ runid < Inf # TODO: update this at the end of 6th campaign
|
|
7
|
|
else
|
|
@error "Fix this function"
|
|
DomainError("runid $(runid) is not registered to the software")
|
|
end
|
|
end
|
|
|
|
"""
|
|
add_qaqc_single_result(
|
|
db::SQLite.DB,
|
|
single_result_table::DataFrame,
|
|
runlist_table::DataFrame,
|
|
) -> nothing
|
|
|
|
Fill `qaqc_single_run_results` in `db` from single result table DataFrame.
|
|
Additionally, it
|
|
1. automatically add `runid` if it's not in `qaqc_runs` table in `db`.
|
|
2. automatically update fields in `qaqc_runs` table.
|
|
"""
|
|
function add_qaqc_single_result(
|
|
db::SQLite.DB,
|
|
single_result_table::DataFrame,
|
|
runlist_table::DataFrame,
|
|
)
|
|
single_result_table = prepare_single_result_df(single_result_table)
|
|
|
|
position_id_map =
|
|
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
|
(i, s) = x
|
|
s => i
|
|
end) |> Dict
|
|
|
|
stmt_search_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
SELECT id
|
|
FROM qaqc_runs
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_insert_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_runs
|
|
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
|
""",
|
|
)
|
|
stmt_update_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
UPDATE qaqc_runs
|
|
SET run_datetime = :run_datetime, shifter = :shifter, logfile = :logfile, shiftscript_ver = :shiftscript_ver
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_insert_result = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO
|
|
qaqc_single_run_results(
|
|
runid,
|
|
psboard_id,
|
|
daughterboard_id,
|
|
position,
|
|
resistance_test_passed,
|
|
qspip,
|
|
recov,
|
|
power,
|
|
clock,
|
|
asdtp,
|
|
reset,
|
|
qaqc_result,
|
|
note
|
|
)
|
|
VALUES (
|
|
:runid,
|
|
:psboard_id,
|
|
:daughterboard_id,
|
|
:position,
|
|
:resistance_test_passed,
|
|
:qspip,
|
|
:recov,
|
|
:power,
|
|
:clock,
|
|
:asdtp,
|
|
:reset,
|
|
:qaqc_result,
|
|
:note
|
|
)
|
|
""",
|
|
)
|
|
stmt_insert_resistance = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_resistance_check(psb_id, passed)
|
|
VALUES (:psboard_id, :ispassed)
|
|
""",
|
|
)
|
|
|
|
for row in eachrow(single_result_table)
|
|
if ismissing(row.runid)
|
|
@assert contains("resistance")(row.comment) || contains("CN15")(row.comment) "Unexpected row with id $(row.motherboard_id) $(row.comment)"
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_resistance,
|
|
(psboard_id = row.motherboard_id, ispassed = false),
|
|
)
|
|
|
|
continue
|
|
end
|
|
|
|
# Add run if it's not in `qaqc_runs` table
|
|
# or update info on the run (such as datetime)
|
|
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
|
|
campaign_id = get_campaign_id_from_run_id(row.runid)
|
|
comment = let
|
|
row_run = filter(
|
|
Symbol("Run ID") => x -> !ismissing(x) && x == row.runid,
|
|
runlist_table,
|
|
)
|
|
if !isempty(row_run)
|
|
row_run.comment
|
|
else
|
|
""
|
|
end
|
|
end
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_runid,
|
|
(
|
|
runid = row.runid,
|
|
campaign_id = campaign_id,
|
|
run_datetime = row.timestamp |> string,
|
|
note = comment,
|
|
shifter = row.shifter,
|
|
logfile = row.qaqc_log_file,
|
|
shiftscript_ver = row.shiftscript_ver,
|
|
),
|
|
)
|
|
else
|
|
DBInterface.execute(
|
|
stmt_update_runid,
|
|
(
|
|
runid = row.runid,
|
|
run_datetime = row.timestamp |> string,
|
|
shifter = row.shifter,
|
|
logfile = row.qaqc_log_file,
|
|
shiftscript_ver = row.shiftscript_ver,
|
|
),
|
|
)
|
|
end
|
|
|
|
# resistance
|
|
DBInterface.execute(
|
|
stmt_insert_resistance,
|
|
(psboard_id = row.motherboard_id, ispassed = true),
|
|
)
|
|
|
|
# main result
|
|
DBInterface.execute(
|
|
stmt_insert_result,
|
|
(
|
|
runid = row.runid,
|
|
psboard_id = row.motherboard_id,
|
|
daughterboard_id = row.daughterboard,
|
|
position = position_id_map[row.position],
|
|
resistance_test_passed = true,
|
|
qspip = row.qspip,
|
|
recov = row.recov,
|
|
power = row.power,
|
|
clock = row.clock,
|
|
asdtp = row.asdtp,
|
|
reset = row.reset,
|
|
qaqc_result = row.qaqc_result,
|
|
note = row.comment,
|
|
),
|
|
)
|
|
end
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame
|
|
|
|
Format `qaqc_dispatch` DataFrame from exported CSV.
|
|
Used in [`add_qaqc_dispatch`](@ref).
|
|
"""
|
|
function prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame
|
|
df = copy(raw_dispatch_table, copycols = true)
|
|
transform!(
|
|
df,
|
|
[Symbol("Column$i") for i in 2:ncol(df)] => ByRow((s...) -> join(s |> skipmissing)) => :comment,
|
|
)
|
|
select!(df, [1, ncol(df)])
|
|
rename!(df, [:psboard_id, :comment])
|
|
|
|
transform!(
|
|
df,
|
|
:psboard_id => (vs -> accumulate(vs; init = 0) do x, y
|
|
ispsbid = startswith("PS")
|
|
if ispsbid(y)
|
|
x
|
|
else
|
|
match(r"(\d+)", y) |> first
|
|
end
|
|
end) => :campaign_id,
|
|
)
|
|
transform!(
|
|
df,
|
|
:psboard_id =>
|
|
ByRow(s -> startswith("PS")(s) ? parse(Int64, s[3:end]) : missing) =>
|
|
:psboard_id,
|
|
)
|
|
dropmissing!(df, :psboard_id)
|
|
|
|
df
|
|
end
|
|
|
|
"""
|
|
add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
|
|
|
Fill `qaqc_dispatch` table in `db` from `dispatch_table`.
|
|
"""
|
|
function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
|
dispatch_table = prepare_dispatch_table(dispatch_table)
|
|
|
|
# TODO: provide datetime
|
|
stmt_insert_dispatch = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_dispatch(qaqc_campaign_id, psb_id, source_place, destination, time)
|
|
VALUES (:campaign_id, :psboard_id, "KEK", "GND", NULL)
|
|
""",
|
|
)
|
|
|
|
DBInterface.executemany(
|
|
stmt_insert_dispatch,
|
|
(campaign_id = dispatch_table.campaign_id, psboard_id = dispatch_table.psboard_id),
|
|
)
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString) -> nothing
|
|
|
|
Add qaqc run list from master log files in `logs_dir`.
|
|
Currently, it adds long runs and run with id 20-23 only (since normal runs are added from single run results table).
|
|
"""
|
|
function add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString)
|
|
stmt_search_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
SELECT id, run_datetime
|
|
FROM qaqc_runs
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_update_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
UPDATE qaqc_runs
|
|
SET run_datetime = :run_datetime, shifter = :shifter, logfile = :logfile, shiftscript_ver = :shiftscript_ver
|
|
WHERE id = :runid
|
|
""",
|
|
)
|
|
stmt_insert_runid = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO qaqc_runs
|
|
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
|
""",
|
|
)
|
|
|
|
is_run_to_add(log_file) = begin
|
|
m = match(r"(\d+)\.log", log_file)
|
|
contains("_long.log")(log_file) ||
|
|
!isnothing(m) && (
|
|
begin
|
|
num = parse(Int64, m[1])
|
|
20 <= num <= 23 || num == 27 || num == 28
|
|
end
|
|
)
|
|
end
|
|
|
|
longrun_logs = readdir(logs_dir; join = true) |> filter(is_run_to_add)
|
|
# longrun_logs = readdir(logs_dir; join = true) |> filter(contains("_long.log"))
|
|
for longrun_log in longrun_logs
|
|
run_metadata = QaqcMasterLog.parse_master_log(longrun_log)
|
|
if isnothing(run_metadata)
|
|
continue
|
|
end
|
|
current_rundb =
|
|
DBInterface.execute(stmt_search_runid, (; runid = run_metadata.runid)) |>
|
|
DataFrame
|
|
if !isempty(current_rundb.id)
|
|
# runid is already in the database
|
|
if any(ismissing, current_rundb.run_datetime)
|
|
# add timestamp, logfile, ...etc
|
|
DBInterface.execute(
|
|
stmt_update_runid,
|
|
(
|
|
runid = run_metadata.runid,
|
|
run_datetime = run_metadata.timestamp |> string,
|
|
shifter = run_metadata.shifters,
|
|
logfile = splitdir(longrun_log) |> last,
|
|
shiftscript_ver = run_metadata.shiftscript_version,
|
|
),
|
|
)
|
|
end
|
|
continue
|
|
end
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_runid,
|
|
(
|
|
runid = run_metadata.runid,
|
|
campaign_id = get_campaign_id_from_run_id(run_metadata.runid),
|
|
run_datetime = run_metadata.timestamp |> string,
|
|
note = "",
|
|
shifter = run_metadata.shifters,
|
|
logfile = splitdir(longrun_log) |> last,
|
|
shiftscript_ver = run_metadata.shiftscript_version,
|
|
),
|
|
)
|
|
end
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
prepare_100test_table(table::DataFrame)::DataFrame
|
|
|
|
Format 100test result `table` from exported CSV.
|
|
Used in [`add_qaqc_100test_result`](@ref).
|
|
|
|
# Detail
|
|
- Format `motherboard ID`s
|
|
- `PS00xxxx` -> `Int64(xxxx)`
|
|
- `xxxx` -> `Int64(xxxx)`
|
|
- For `psbid == 484` and `runid == 115` results, make all result fields to `missing` since they contain abnormal strings.
|
|
"""
|
|
function prepare_100test_table(table::DataFrame)::DataFrame
|
|
df = copy(table, copycols = true)
|
|
|
|
transform!(
|
|
df,
|
|
Symbol("motherboard ID") =>
|
|
ByRow(s -> if startswith("PS")(s)
|
|
parse(Int64, s[3:end])
|
|
else
|
|
parse(Int64, s)
|
|
end) => :motherboard_id,
|
|
)
|
|
transform!(
|
|
df,
|
|
Cols(:motherboard_id, :runid, 4:13) =>
|
|
ByRow((psbid, runid, items...) -> if psbid == 484 && runid == 115
|
|
missings(items |> length)
|
|
else
|
|
items
|
|
end) => names(df)[4:13],
|
|
)
|
|
|
|
df
|
|
end
|
|
|
|
"""
|
|
get_num_tests_for_extra_runs(runid::Int64)
|
|
|
|
Get number of tests for extra QAQC runs.
|
|
They are usually 100.
|
|
|
|
Current abnormal runs:
|
|
|
|
| runid | # of runs |
|
|
|-------|-----------|
|
|
| 99| 246|
|
|
"""
|
|
function get_num_tests_for_extra_runs(runid::Int64)
|
|
if runid == 99
|
|
246
|
|
else
|
|
100
|
|
end
|
|
end
|
|
|
|
"""
|
|
add_qaqc_100test_result(
|
|
db::SQLite.DB,
|
|
table::DataFrame,
|
|
logs_dir::AbstractString,
|
|
jld2_slavelog_path::Union{AbstractString, Nothing},
|
|
) -> nothing
|
|
|
|
Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
|
|
which is converted from a raw exported CSV.
|
|
|
|
# Args
|
|
|
|
- `table`: 100 test result table, prepared with [`prepare_100test_table`](@ref)
|
|
- `logs_dir`: where slave log files located in certain format
|
|
|
|
# Detail
|
|
- skips psboards in `resistance_test_passed` with `passed == false`
|
|
"""
|
|
function add_qaqc_100test_result(
|
|
db::SQLite.DB,
|
|
table::DataFrame,
|
|
logs_dir::AbstractString,
|
|
jld2_slavelog::Union{JLD2.JLDFile, Nothing},
|
|
)
|
|
position_id_map =
|
|
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
|
(i, s) = x
|
|
s => i
|
|
end) |> Dict
|
|
|
|
table = prepare_100test_table(table)
|
|
|
|
qaqc_run_ids =
|
|
DBInterface.execute(
|
|
db,
|
|
sql"""
|
|
SELECT id
|
|
FROM qaqc_runs
|
|
""",
|
|
) |> Tables.columntable |> (t -> t.id)
|
|
resistance_error_psb_list =
|
|
DBInterface.execute(
|
|
db,
|
|
sql"""
|
|
SELECT psb_id
|
|
FROM qaqc_resistance_check
|
|
WHERE passed = 0
|
|
""",
|
|
) |>
|
|
Tables.columntable |>
|
|
(t -> t.psb_id)
|
|
stmt_insert_result = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
INSERT INTO
|
|
qaqc_extra_run_results (
|
|
runid,
|
|
psboard_id,
|
|
position,
|
|
num_tests,
|
|
insufficient_reset_with_10,
|
|
reset_failed_though_reconfig_done,
|
|
always_hit_flag_true,
|
|
dac_is_0,
|
|
bcid_shift,
|
|
efficiency_99percent,
|
|
bcid_fail_111,
|
|
bcid_fail_000,
|
|
low_efficiency,
|
|
bcid_fail,
|
|
invalid_register_value,
|
|
power_out_of_range,
|
|
note,
|
|
is_slavelog_valid
|
|
)
|
|
VALUES (
|
|
:runid,
|
|
:psboard_id,
|
|
:position,
|
|
:num_tests,
|
|
:insufficient_reset_with_10,
|
|
:reset_failed_though_reconfig_done,
|
|
:always_hit_flag_true,
|
|
:dac_is_0,
|
|
:bcid_shift,
|
|
:efficiency_99percent,
|
|
:bcid_fail_111,
|
|
:bcid_fail_000,
|
|
:low_efficiency,
|
|
:bcid_fail,
|
|
:invalid_register_value,
|
|
:power_out_of_range,
|
|
:note,
|
|
:is_slavelog_valid
|
|
)
|
|
""",
|
|
)
|
|
lock_db = ReentrantLock()
|
|
lock_jld2 = ReentrantLock()
|
|
|
|
Threads.@threads for row in eachrow(table)
|
|
if ismissing(row.runid) || !(row.runid in qaqc_run_ids)
|
|
# search for resistance error
|
|
if row.motherboard_id in resistance_error_psb_list
|
|
continue
|
|
end
|
|
error("Runid $(row.runid) not found in `qaqc_runs` table.")
|
|
end
|
|
|
|
slavelog_filename = "$(row.motherboard_id)_$(row.runid)_longrun.txt"
|
|
is_slavelog_valid, slavelog_result = try
|
|
result = SlaveLogParser.parse_slavelog_file(
|
|
joinpath(logs_dir, "main", slavelog_filename),
|
|
)
|
|
true, result
|
|
catch e
|
|
@debug "Failed to parse slave log due to $(e)" catch_backtrace()
|
|
false, missing
|
|
end
|
|
|
|
lock(lock_db) do
|
|
DBInterface.execute(
|
|
stmt_insert_result,
|
|
(
|
|
runid = row.runid,
|
|
psboard_id = row.motherboard_id,
|
|
position = position_id_map[row.position],
|
|
num_tests = get_num_tests_for_extra_runs(row.runid),
|
|
insufficient_reset_with_10 = row.var"10回reset足りず",
|
|
reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない",
|
|
always_hit_flag_true = row.var"always_hit_flag",
|
|
dac_is_0 = row.var"DAC = 0",
|
|
bcid_shift = row.var"DAC = 0",
|
|
efficiency_99percent = row.var"efficiency 99%",
|
|
bcid_fail_111 = row.var"BCID 0:0:0",
|
|
bcid_fail_000 = row.var"BCID 1:1:1",
|
|
low_efficiency = row.var"low efficiency",
|
|
bcid_fail = row.var"BCID fail",
|
|
invalid_register_value = row.var"invalid register values",
|
|
power_out_of_range = row.var"power out of range",
|
|
note = row.Column20,
|
|
is_slavelog_valid,
|
|
),
|
|
)
|
|
end
|
|
|
|
lock(lock_jld2) do
|
|
if !isnothing(jld2_slavelog)
|
|
if haskey(jld2_slavelog, slavelog_filename)
|
|
@debug "slave log already included: $(slavelog_filename)"
|
|
else
|
|
jld2_slavelog[slavelog_filename] = slavelog_result
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
|
|
|
|
Insert skew measurement result from slave logs with name `psbid_runid_clk.txt`.
|
|
See [`ClockParser.get_skew`](@ref) for parse detail.
|
|
|
|
# Abnormal logs:
|
|
- `48_nagoya_irradition_...`: skipped
|
|
- `630_190`: broken file
|
|
"""
|
|
function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
|
|
stmt_insert_skew_to_single_result = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
UPDATE qaqc_single_run_results
|
|
SET lvds_tx_skew = :skew, is_slaveclocklog_valid = :is_slaveclocklog_valid
|
|
WHERE runid = :runid AND psboard_id = :psbid
|
|
""",
|
|
)
|
|
clk_files =
|
|
readdir(joinpath(logs_dir, "main"), join = true) |>
|
|
filter(endswith("_clk.txt")) |>
|
|
filter(!contains("nagoya"))
|
|
|
|
DBInterface.transaction(db) do
|
|
for file in clk_files
|
|
m = match(r"^(?<psbid>\d+)_(?<runid>\d+)_clk.txt$", splitdir(file) |> last)
|
|
if isnothing(m)
|
|
error("Invalid filename $(file)")
|
|
end
|
|
|
|
if m[:psbid] == "630" && m[:runid] == "190"
|
|
@debug "skipping... (psbid=630 runid=190 is broken)"
|
|
DBInterface.execute(
|
|
stmt_insert_skew_to_single_result,
|
|
(
|
|
skew = missing,
|
|
is_slaveclocklog_valid = false,
|
|
runid = m[:runid],
|
|
psbid = m[:psbid],
|
|
),
|
|
)
|
|
continue
|
|
elseif m[:psbid] == "627" && m[:runid] == "344"
|
|
@debug "skipping... (psbid=627 runid=344 is broken)"
|
|
DBInterface.execute(
|
|
stmt_insert_skew_to_single_result,
|
|
(
|
|
skew = missing,
|
|
is_slaveclocklog_valid = false,
|
|
runid = m[:runid],
|
|
psbid = m[:psbid],
|
|
),
|
|
)
|
|
continue
|
|
end
|
|
|
|
skew = try
|
|
ClockParser.get_skew(file)
|
|
catch e
|
|
@error "Failed to parse clock skew: psbid: $(m[:psbid]), runid: $(m[:runid])" file
|
|
throw(e)
|
|
end
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_skew_to_single_result,
|
|
(
|
|
skew = ClockParser.get_skew(file),
|
|
is_slaveclocklog_valid = true,
|
|
runid = m[:runid],
|
|
psbid = m[:psbid],
|
|
),
|
|
)
|
|
end
|
|
end
|
|
|
|
nothing
|
|
end
|
|
|
|
"""
|
|
add_slavelog_result(
|
|
db::SQLite.DB,
|
|
logs_dir::AbstractString,
|
|
jld2_slavelog::Union{Nothing, JLD2.JLDFile},
|
|
)
|
|
|
|
Extract QAQC results from slave log files for single runs.
|
|
Slave log files are expected to located in certain format under `logs_dir`.
|
|
"""
|
|
function add_slavelog_result(
|
|
db::SQLite.DB,
|
|
logs_dir::AbstractString,
|
|
jld2_slavelog::Union{Nothing, JLD2.JLDFile},
|
|
)
|
|
exclude_runs = (
|
|
(runid = 51, psbid = nothing, reason = "clock only"),
|
|
(runid = 175, psbid = nothing, reason = "broken files"),
|
|
(runid = 437, psbid = 1215, reason = "PSBID 1215 is not completed"), # debug 6.5
|
|
(runid = 439, psbid = 703, reason = "PSBID 703 is not completed"), # debug 6.5
|
|
(runid = 434, psbid = 723, reason = "PSBID 723 is not completed"),
|
|
)
|
|
@assert eltype(exclude_runs) != Any
|
|
|
|
stmt_insert_slave_result_to_single_result = DBInterface.prepare(
|
|
db,
|
|
sql"""
|
|
UPDATE qaqc_single_run_results
|
|
SET
|
|
power_3v3d = :power_3v3d,
|
|
power_3v3a = :power_3v3a,
|
|
power_n3va = :power_n3va,
|
|
is_slavelog_valid = :is_slavelog_valid
|
|
WHERE
|
|
runid = :runid AND psboard_id = :psbid
|
|
""",
|
|
)
|
|
runids =
|
|
DBInterface.execute(
|
|
db,
|
|
sql"""
|
|
SELECT id
|
|
FROM qaqc_runs
|
|
""",
|
|
) |> Tables.columntable |> (tbl -> tbl.id)
|
|
|
|
slave_file_paths =
|
|
readdir(joinpath(logs_dir, "main"), join = true) |>
|
|
filter(contains(r"\d+_\d+\.txt"))
|
|
|
|
DBInterface.transaction(db) do
|
|
for file_path in slave_file_paths
|
|
psbid, runid, islongrun =
|
|
SlaveLogParser.get_psbid_runid_from_filename(basename(file_path))
|
|
@assert !islongrun
|
|
|
|
# exclusion
|
|
|
|
exclude_cond = filter(exclude_runs) do cond
|
|
runid_matched = runid == cond.runid
|
|
psbid_matched = isnothing(cond.psbid) || cond.psbid == psbid
|
|
runid_matched && psbid_matched
|
|
end
|
|
if !isempty(exclude_cond)
|
|
@debug "skipping runid = $(runid) for $(first(exclude_cond).reason)"
|
|
DBInterface.execute(
|
|
stmt_insert_slave_result_to_single_result,
|
|
(;
|
|
power_3v3d = missing,
|
|
power_3v3a = missing,
|
|
power_n3va = missing,
|
|
is_slavelog_valid = false,
|
|
runid,
|
|
psbid,
|
|
),
|
|
)
|
|
continue
|
|
end
|
|
|
|
if !(runid in runids)
|
|
@debug "runid: $(runid) not in run list (psbid: $(psbid)). Parsing slave log to test its format."
|
|
slave_result = SlaveLogParser.parse_slavelog_file(file_path)
|
|
continue
|
|
end
|
|
|
|
# main
|
|
|
|
slave_result = try
|
|
SlaveLogParser.parse_slavelog_file(file_path)
|
|
catch e
|
|
throw(error("Failed to parse slave log file: $(file_path)\n$(e)"))
|
|
end
|
|
|
|
@assert length(slave_result.power) == 1 "Too many power results for single run"
|
|
|
|
DBInterface.execute(
|
|
stmt_insert_slave_result_to_single_result,
|
|
(;
|
|
power_3v3d = slave_result.power[1].result_3v3d,
|
|
power_3v3a = slave_result.power[1].result_3v3a,
|
|
power_n3va = slave_result.power[1].result_n3va,
|
|
is_slavelog_valid = true,
|
|
runid,
|
|
psbid,
|
|
),
|
|
)
|
|
|
|
if !isnothing(jld2_slavelog)
|
|
jld2_slavelog[basename(file_path)] = slave_result
|
|
end
|
|
end
|
|
end
|
|
|
|
nothing
|
|
end
|