mirror of
https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git
synced 2025-06-10 14:59:23 +09:00
add: import 100 run results + more
- master log parser to get run info - test for master log parser - master log reader - note field for single run - fix datetime manipulation for single result table
This commit is contained in:
parent
f5e6caf936
commit
21f38e83db
9 changed files with 1505 additions and 16 deletions
|
|
@ -3,13 +3,43 @@ module PSBoardDataBase
|
|||
using SQLite
|
||||
using DBInterface
|
||||
using Tables
|
||||
using CSV
|
||||
using DataFrames
|
||||
using Dates
|
||||
|
||||
include("parse_qaqc_master_log.jl")
|
||||
|
||||
include("create_table.jl")
|
||||
|
||||
include("import_data.jl")
|
||||
|
||||
function create_database_from_exported_csvs(
|
||||
dbpath::AbstractString;
|
||||
single_run_csv::AbstractString,
|
||||
runlist_csv::AbstractString,
|
||||
dispatch_csv::AbstractString,
|
||||
hundred_csv::AbstractString,
|
||||
masterlog_dir::AbstractString,
|
||||
)
|
||||
db = create_database(dbpath)
|
||||
single_result_df = CSV.read(single_run_csv, DataFrame)
|
||||
runlist_table = CSV.read(runlist_csv, DataFrame)
|
||||
dispatch_table = CSV.read(dispatch_csv, DataFrame)
|
||||
extra_100test_result_df = CSV.read(hundred_csv, DataFrame)
|
||||
|
||||
insert_qaqc_campaign_id(db)
|
||||
insert_qaqc_positions(db)
|
||||
|
||||
add_psboard_ids(db, single_result_df)
|
||||
add_qaqc_runlist(db, runlist_table)
|
||||
add_qaqc_single_result(db, single_result_df, runlist_table)
|
||||
add_qaqc_dispatch(db, dispatch_table)
|
||||
add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
|
||||
add_qaqc_100test_result(db, extra_100test_result_df)
|
||||
|
||||
db
|
||||
end
|
||||
|
||||
greet() = print("Hello World!")
|
||||
|
||||
end # module PSBoardDataBase
|
||||
|
|
|
|||
|
|
@ -130,11 +130,23 @@ function add_qaqc_runlist(db::SQLite.DB, runlist_table::DataFrame)
|
|||
nothing
|
||||
end
|
||||
|
||||
function get_campaign_id_from_run_id(runid::Integer)
|
||||
if runid < 63
|
||||
1
|
||||
elseif runid < 98
|
||||
2
|
||||
else
|
||||
3
|
||||
end
|
||||
end
|
||||
|
||||
function add_qaqc_single_result(
|
||||
db::SQLite.DB,
|
||||
single_result_table::DataFrame,
|
||||
runlist_table::DataFrame,
|
||||
)
|
||||
single_result_table = prepare_single_result_df(single_result_table)
|
||||
|
||||
position_id_map =
|
||||
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
||||
(i, s) = x
|
||||
|
|
@ -180,7 +192,8 @@ function add_qaqc_single_result(
|
|||
clock,
|
||||
asdtp,
|
||||
reset,
|
||||
qaqc_result
|
||||
qaqc_result,
|
||||
note
|
||||
)
|
||||
VALUES (
|
||||
:runid,
|
||||
|
|
@ -194,7 +207,8 @@ function add_qaqc_single_result(
|
|||
:clock,
|
||||
:asdtp,
|
||||
:reset,
|
||||
:qaqc_result
|
||||
:qaqc_result,
|
||||
:note
|
||||
)
|
||||
""",
|
||||
)
|
||||
|
|
@ -221,13 +235,7 @@ function add_qaqc_single_result(
|
|||
# Add run if it's not in `qaqc_runs` table
|
||||
# or update info on the run (such as datetime)
|
||||
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
|
||||
campaign_id = if row.runid < 63
|
||||
1
|
||||
elseif row.runid < 98
|
||||
2
|
||||
else
|
||||
3
|
||||
end
|
||||
campaign_id = get_campaign_id_from_run_id(row.runid)
|
||||
comment = let
|
||||
row_run = filter(
|
||||
Symbol("Run ID") => x -> !ismissing(x) && x == row.runid,
|
||||
|
|
@ -245,7 +253,7 @@ function add_qaqc_single_result(
|
|||
(
|
||||
runid = row.runid,
|
||||
campaign_id = campaign_id,
|
||||
run_datetime = row.timestamp,
|
||||
run_datetime = row.timestamp |> string,
|
||||
note = comment,
|
||||
shifter = row.shifter,
|
||||
logfile = row.qaqc_log_file,
|
||||
|
|
@ -257,7 +265,7 @@ function add_qaqc_single_result(
|
|||
stmt_update_runid,
|
||||
(
|
||||
runid = row.runid,
|
||||
run_datetime = row.timestamp,
|
||||
run_datetime = row.timestamp |> string,
|
||||
shifter = row.shifter,
|
||||
logfile = row.qaqc_log_file,
|
||||
shiftscript_ver = row.shiftscript_ver,
|
||||
|
|
@ -287,6 +295,7 @@ function add_qaqc_single_result(
|
|||
asdtp = row.asdtp,
|
||||
reset = row.reset,
|
||||
qaqc_result = row.qaqc_result,
|
||||
note = row.comment,
|
||||
),
|
||||
)
|
||||
end
|
||||
|
|
@ -325,7 +334,7 @@ function prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame
|
|||
df
|
||||
end
|
||||
|
||||
function add_qaqcdispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
||||
function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
||||
dispatch_table = prepare_dispatch_table(dispatch_table)
|
||||
|
||||
# TODO: provide datetime
|
||||
|
|
@ -344,3 +353,197 @@ function add_qaqcdispatch(db::SQLite.DB, dispatch_table::DataFrame)
|
|||
|
||||
nothing
|
||||
end
|
||||
|
||||
"""
|
||||
Add qaqc run list from master log files in `logs_dir`.
|
||||
Currently, it adds long runs and run with id 20-23 only (since normal runs are added from single run results table).
|
||||
"""
|
||||
function add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString)
|
||||
stmt_search_runid = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
SELECT id
|
||||
FROM qaqc_runs
|
||||
WHERE id = :runid
|
||||
""",
|
||||
)
|
||||
stmt_insert_runid = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
INSERT INTO qaqc_runs
|
||||
VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
|
||||
""",
|
||||
)
|
||||
|
||||
is_run_to_add(log_file) = begin
|
||||
m = match(r"(\d+)\.log", log_file)
|
||||
contains("_long.log")(log_file) ||
|
||||
!isnothing(m) && (
|
||||
begin
|
||||
num = parse(Int64, m[1])
|
||||
20 <= num <= 23 || num == 27 || num == 28
|
||||
end
|
||||
)
|
||||
end
|
||||
|
||||
longrun_logs = readdir(logs_dir; join = true) |> filter(is_run_to_add)
|
||||
# longrun_logs = readdir(logs_dir; join = true) |> filter(contains("_long.log"))
|
||||
for longrun_log in longrun_logs
|
||||
run_metadata = QaqcMasterLog.parse_master_log(longrun_log)
|
||||
if isnothing(run_metadata)
|
||||
continue
|
||||
end
|
||||
|
||||
if !isempty(DBInterface.execute(stmt_search_runid, (; runid = run_metadata.runid)))
|
||||
# runid is already in the database
|
||||
continue
|
||||
end
|
||||
|
||||
DBInterface.execute(
|
||||
stmt_insert_runid,
|
||||
(
|
||||
runid = run_metadata.runid,
|
||||
campaign_id = get_campaign_id_from_run_id(run_metadata.runid),
|
||||
run_datetime = run_metadata.timestamp,
|
||||
note = "",
|
||||
shifter = run_metadata.shifters,
|
||||
logfile = splitdir(longrun_log) |> last,
|
||||
shiftscript_ver = run_metadata.shiftscript_version,
|
||||
),
|
||||
)
|
||||
end
|
||||
|
||||
nothing
|
||||
end
|
||||
|
||||
function prepare_100test_table(table::DataFrame)::DataFrame
|
||||
df = copy(table, copycols = true)
|
||||
|
||||
transform!(
|
||||
df,
|
||||
Symbol("motherboard ID") =>
|
||||
ByRow(s -> if startswith("PS")(s)
|
||||
parse(Int64, s[3:end])
|
||||
else
|
||||
parse(Int64, s)
|
||||
end) => :motherboard_id,
|
||||
)
|
||||
transform!(
|
||||
df,
|
||||
Cols(:motherboard_id, :runid, 4:13) =>
|
||||
ByRow((psbid, runid, items...) -> if psbid == 484 && runid == 115
|
||||
missings(items |> length)
|
||||
else
|
||||
items
|
||||
end) => names(df)[4:13],
|
||||
)
|
||||
|
||||
df
|
||||
end
|
||||
|
||||
function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
|
||||
position_id_map =
|
||||
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
|
||||
(i, s) = x
|
||||
s => i
|
||||
end) |> Dict
|
||||
|
||||
table = prepare_100test_table(table)
|
||||
|
||||
stmt_search_runid = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
SELECT id
|
||||
FROM qaqc_runs
|
||||
WHERE id = :runid
|
||||
""",
|
||||
)
|
||||
stmt_search_resistance_error = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
SELECT psb_id
|
||||
FROM qaqc_resistance_check
|
||||
WHERE psb_id = :psboard_id AND passed = 0
|
||||
""",
|
||||
)
|
||||
stmt_insert_result = DBInterface.prepare(
|
||||
db,
|
||||
sql"""
|
||||
INSERT INTO
|
||||
qaqc_extra_run_results (
|
||||
runid,
|
||||
psboard_id,
|
||||
position,
|
||||
num_tests,
|
||||
insufficient_reset_with_10,
|
||||
reset_failed_though_reconfig_done,
|
||||
always_hit_flag_true,
|
||||
dac_is_0,
|
||||
bcid_shift,
|
||||
efficiency_99percent,
|
||||
bcid_fail_111,
|
||||
bcid_fail_000,
|
||||
low_efficiency,
|
||||
bcid_fail,
|
||||
invalid_register_value,
|
||||
power_out_of_range,
|
||||
note
|
||||
)
|
||||
VALUES (
|
||||
:runid,
|
||||
:psboard_id,
|
||||
:position,
|
||||
100,
|
||||
:insufficient_reset_with_10,
|
||||
:reset_failed_though_reconfig_done,
|
||||
:always_hit_flag_true,
|
||||
:dac_is_0,
|
||||
:bcid_shift,
|
||||
:efficiency_99percent,
|
||||
:bcid_fail_111,
|
||||
:bcid_fail_000,
|
||||
:low_efficiency,
|
||||
:bcid_fail,
|
||||
:invalid_register_value,
|
||||
:power_out_of_range,
|
||||
:note
|
||||
)
|
||||
""",
|
||||
)
|
||||
|
||||
for row in eachrow(table)
|
||||
# TODO: get runid from master log file
|
||||
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
|
||||
# search for resistance error
|
||||
if !isempty(
|
||||
DBInterface.execute(stmt_search_resistance_error, (; psboard_id = row.motherboard_id)),
|
||||
)
|
||||
continue
|
||||
end
|
||||
end
|
||||
|
||||
DBInterface.execute(
|
||||
stmt_insert_result,
|
||||
(
|
||||
runid = row.runid,
|
||||
psboard_id = row.motherboard_id,
|
||||
position = position_id_map[row.position],
|
||||
insufficient_reset_with_10 = row.var"10回reset足りず",
|
||||
reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない",
|
||||
always_hit_flag_true = row.var"always_hit_flag",
|
||||
dac_is_0 = row.var"DAC = 0",
|
||||
bcid_shift = row.var"DAC = 0",
|
||||
efficiency_99percent = row.var"efficiency 99%",
|
||||
bcid_fail_111 = row.var"BCID 0:0:0",
|
||||
bcid_fail_000 = row.var"BCID 1:1:1",
|
||||
low_efficiency = row.var"low efficiency",
|
||||
bcid_fail = row.var"BCID fail",
|
||||
invalid_register_value = row.var"invalid register values",
|
||||
power_out_of_range = row.var"power out of range",
|
||||
note = row.Column20,
|
||||
),
|
||||
)
|
||||
end
|
||||
|
||||
nothing
|
||||
end
|
||||
|
|
|
|||
56
src/parse_qaqc_master_log.jl
Normal file
56
src/parse_qaqc_master_log.jl
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
module QaqcMasterLog
|
||||
|
||||
using Dates
|
||||
|
||||
"""
|
||||
Metadata(not results) of QAQC run from master log.
|
||||
|
||||
# Fields
|
||||
- `timestamp`: in UTC
|
||||
- `runid`
|
||||
- `shifters`
|
||||
- `shiftscript_version::VersionNumber`
|
||||
"""
|
||||
struct QaqcMasterLogMetadata
|
||||
timestamp::DateTime
|
||||
runid::Int64
|
||||
shifters::String
|
||||
shiftscript_version::VersionNumber
|
||||
end
|
||||
|
||||
"""
|
||||
Parse master log.
|
||||
If the `logfile` is empty, return `nothing`.
|
||||
"""
|
||||
function parse_master_log(logfile::AbstractString)
|
||||
open(logfile) do f
|
||||
first_line = readline(f)
|
||||
if isempty(first_line)
|
||||
@debug "file $(logfile) is empty"
|
||||
@assert read(logfile) |> isempty
|
||||
return nothing
|
||||
end
|
||||
shiftscript_version = split(first_line) |> last |> VersionNumber
|
||||
|
||||
@assert readline(f) |> contains("-----")
|
||||
|
||||
date_line = readline(f)
|
||||
@assert date_line |> split |> first |> contains("Date")
|
||||
date_part = split(date_line) |> last
|
||||
datetime, timezone = split(date_part, '+')
|
||||
@assert timezone == "0000" "Unexpected timestamp: timezone is not UTC"
|
||||
# Maybe use ZonedDateTime here
|
||||
timestamp = datetime |> DateTime
|
||||
|
||||
runid_line = readline(f)
|
||||
runid = parse(Int64, runid_line |> split |> last)
|
||||
|
||||
shifters_line = readline(f)
|
||||
@assert shifters_line |> split |> first |> contains("Shifters")
|
||||
shifters = shifters_line |> split |> last
|
||||
|
||||
return QaqcMasterLogMetadata(timestamp, runid, shifters, shiftscript_version)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
|
@ -17,6 +17,7 @@ CREATE TABLE qaqc_single_run_results (
|
|||
asdtp INTEGER,
|
||||
reset INTEGER,
|
||||
qaqc_result INTEGER,
|
||||
note TEXT,
|
||||
FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
|
||||
FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
|
||||
FOREIGN KEY("position") REFERENCES "qaqc_positions"("id")
|
||||
|
|
@ -63,11 +64,27 @@ CREATE TABLE qaqc_extra_run_results (
|
|||
runid INTEGER,
|
||||
psboard_id INTEGER NOT NULL,
|
||||
position INTEGER,
|
||||
num_tests INTEGER,
|
||||
insufficient_reset_with_10 INTEGER,
|
||||
reset_failed_though_reconfig_done INTEGER,
|
||||
always_hit_flag_true INTEGER,
|
||||
dac_is_0 INTEGER,
|
||||
bcid_shift INTEGER,
|
||||
efficiency_99percent INTEGER,
|
||||
bcid_fail_111 INTEGER,
|
||||
bcid_fail_000 INTEGER,
|
||||
low_efficiency INTEGER,
|
||||
bcid_fail INTEGER,
|
||||
invalid_register_value INTEGER,
|
||||
power_out_of_range INTEGER,
|
||||
note TEXT,
|
||||
FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
|
||||
FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
|
||||
FOREIGN KEY("position") REFERENCES "qaqc_positions"("id")
|
||||
);
|
||||
|
||||
-- TODO: add table for desciptions of each error?
|
||||
|
||||
CREATE TABLE qaqc_positions (
|
||||
id INTEGER NOT NULL PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
|
|
@ -98,4 +115,5 @@ AS
|
|||
qaqc_runs
|
||||
WHERE
|
||||
qaqc_positions.id = qaqc_single_run_results.position AND
|
||||
qaqc_runs.id = qaqc_single_run_results.runid
|
||||
qaqc_runs.id = qaqc_single_run_results.runid;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue