From 559f0e5ce70f41ef1cfdbef0997a3e9e31a09b84 Mon Sep 17 00:00:00 2001 From: Wataru Otsubo Date: Wed, 22 Jan 2025 17:47:31 +0100 Subject: [PATCH] new(database column): add slavelog validity to single & extra run results --- CHANGELOG.md | 1 + src/PSBoardDataBase.jl | 2 +- src/import_data.jl | 169 ++++++++++++++++++++++++++------------- src/sql/create_table.sql | 3 + test/runtests.jl | 2 +- 5 files changed, 120 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c90c193..48ad1f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - QAQC progress plot notebook - (notebook) PLLLD wait time scan - Added QAQC campaign 7 to the database +- Added slave log validity columns to single and extra run result tables (!1418) ### Changed diff --git a/src/PSBoardDataBase.jl b/src/PSBoardDataBase.jl index cb98e4c..2d8e6da 100644 --- a/src/PSBoardDataBase.jl +++ b/src/PSBoardDataBase.jl @@ -88,7 +88,7 @@ function create_database_from_exported_csvs( add_qaqc_single_result(db, single_result_df, runlist_table) add_qaqc_dispatch(db, dispatch_table) add_qaqc_runlist_from_masterlogs(db, masterlog_dir) - add_qaqc_100test_result(db, extra_100test_result_df) + add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir) add_skew_from_slave_clk_logs(db, slavelog_dir) add_slavelog_result(db, slavelog_dir) diff --git a/src/import_data.jl b/src/import_data.jl index dc4d0cd..d2f2609 100644 --- a/src/import_data.jl +++ b/src/import_data.jl @@ -608,15 +608,20 @@ function get_num_tests_for_extra_runs(runid::Int64) end """ - add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) -> nothing + add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) -> nothing Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame, which is converted from a raw exported CSV. +# Args + +- `table`: 100 test result table, prepared with [`prepare_100test_table`](@ref) +- `logs_dir`: where slave log files located in certain format + # Detail - skips psboards in `resistance_test_passed` with `passed == false` """ -function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) +function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) position_id_map = ["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin (i, s) = x @@ -625,22 +630,25 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) table = prepare_100test_table(table) - stmt_search_runid = DBInterface.prepare( - db, - sql""" - SELECT id - FROM qaqc_runs - WHERE id = :runid - """, - ) - stmt_search_resistance_error = DBInterface.prepare( - db, - sql""" - SELECT psb_id - FROM qaqc_resistance_check - WHERE psb_id = :psboard_id AND passed = 0 - """, - ) + qaqc_run_ids = + DBInterface.execute( + db, + sql""" + SELECT id + FROM qaqc_runs + """, + ) |> Tables.columntable |> (t -> t.id) + resistance_error_psb_list = + DBInterface.execute( + db, + sql""" + SELECT psb_id + FROM qaqc_resistance_check + WHERE passed = 0 + """, + ) |> + Tables.columntable |> + (t -> t.psb_id) stmt_insert_result = DBInterface.prepare( db, sql""" @@ -662,7 +670,8 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) bcid_fail, invalid_register_value, power_out_of_range, - note + note, + is_slavelog_valid ) VALUES ( :runid, @@ -681,47 +690,61 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) :bcid_fail, :invalid_register_value, :power_out_of_range, - :note + :note, + :is_slavelog_valid ) """, ) + lock_db = ReentrantLock() - for row in eachrow(table) - if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty + Threads.@threads for row in eachrow(table) + if ismissing(row.runid) || !(row.runid in qaqc_run_ids) # search for resistance error - if !isempty( - DBInterface.execute( - stmt_search_resistance_error, - (; psboard_id = row.motherboard_id), - ), - ) + if row.motherboard_id in resistance_error_psb_list continue end error("Runid $(row.runid) not found in `qaqc_runs` table.") end - DBInterface.execute( - stmt_insert_result, - ( - runid = row.runid, - psboard_id = row.motherboard_id, - position = position_id_map[row.position], - num_tests = get_num_tests_for_extra_runs(row.runid), - insufficient_reset_with_10 = row.var"10回reset足りず", - reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない", - always_hit_flag_true = row.var"always_hit_flag", - dac_is_0 = row.var"DAC = 0", - bcid_shift = row.var"DAC = 0", - efficiency_99percent = row.var"efficiency 99%", - bcid_fail_111 = row.var"BCID 0:0:0", - bcid_fail_000 = row.var"BCID 1:1:1", - low_efficiency = row.var"low efficiency", - bcid_fail = row.var"BCID fail", - invalid_register_value = row.var"invalid register values", - power_out_of_range = row.var"power out of range", - note = row.Column20, - ), - ) + is_slavelog_valid = try + SlaveLogParser.parse_slavelog_file( + joinpath( + logs_dir, + "main", + "$(row.motherboard_id)_$(row.runid)_longrun.txt", + ), + ) + true + catch e + @debug "Failed to parse slave log due to $(e)" catch_backtrace() + false + end + + lock(lock_db) do + DBInterface.execute( + stmt_insert_result, + ( + runid = row.runid, + psboard_id = row.motherboard_id, + position = position_id_map[row.position], + num_tests = get_num_tests_for_extra_runs(row.runid), + insufficient_reset_with_10 = row.var"10回reset足りず", + reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない", + always_hit_flag_true = row.var"always_hit_flag", + dac_is_0 = row.var"DAC = 0", + bcid_shift = row.var"DAC = 0", + efficiency_99percent = row.var"efficiency 99%", + bcid_fail_111 = row.var"BCID 0:0:0", + bcid_fail_000 = row.var"BCID 1:1:1", + low_efficiency = row.var"low efficiency", + bcid_fail = row.var"BCID fail", + invalid_register_value = row.var"invalid register values", + power_out_of_range = row.var"power out of range", + note = row.Column20, + is_slavelog_valid, + ), + ) + end end nothing @@ -742,7 +765,7 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) db, sql""" UPDATE qaqc_single_run_results - SET lvds_tx_skew = :skew + SET lvds_tx_skew = :skew, is_slaveclocklog_valid = :is_slaveclocklog_valid WHERE runid = :runid AND psboard_id = :psbid """, ) @@ -760,9 +783,27 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) if m[:psbid] == "630" && m[:runid] == "190" @debug "skipping... (psbid=630 runid=190 is broken)" + DBInterface.execute( + stmt_insert_skew_to_single_result, + ( + skew = missing, + is_slaveclocklog_valid = false, + runid = m[:runid], + psbid = m[:psbid], + ), + ) continue elseif m[:psbid] == "627" && m[:runid] == "344" @debug "skipping... (psbid=627 runid=344 is broken)" + DBInterface.execute( + stmt_insert_skew_to_single_result, + ( + skew = missing, + is_slaveclocklog_valid = false, + runid = m[:runid], + psbid = m[:psbid], + ), + ) continue end @@ -775,7 +816,12 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) DBInterface.execute( stmt_insert_skew_to_single_result, - (skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]), + ( + skew = ClockParser.get_skew(file), + is_slaveclocklog_valid = true, + runid = m[:runid], + psbid = m[:psbid], + ), ) end end @@ -793,8 +839,8 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) exclude_runs = ( (runid = 51, psbid = nothing, reason = "clock only"), (runid = 175, psbid = nothing, reason = "broken files"), - (runid = 437, psbid = 1215, reason = "PSBID 1215 is not completed"), - (runid = 439, psbid = 703, reason = "PSBID 703 is not completed"), + (runid = 437, psbid = 1215, reason = "PSBID 1215 is not completed"), # debug 6.5 + (runid = 439, psbid = 703, reason = "PSBID 703 is not completed"), # debug 6.5 (runid = 434, psbid = 723, reason = "PSBID 723 is not completed"), ) @assert eltype(exclude_runs) != Any @@ -806,7 +852,8 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) SET power_3v3d = :power_3v3d, power_3v3a = :power_3v3a, - power_n3va = :power_n3va + power_n3va = :power_n3va, + is_slavelog_valid = :is_slavelog_valid WHERE runid = :runid AND psboard_id = :psbid """, @@ -839,6 +886,17 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) end if !isempty(exclude_cond) @debug "skipping runid = $(runid) for $(first(exclude_cond).reason)" + DBInterface.execute( + stmt_insert_slave_result_to_single_result, + (; + power_3v3d = missing, + power_3v3a = missing, + power_n3va = missing, + is_slavelog_valid = false, + runid, + psbid, + ), + ) continue end @@ -864,6 +922,7 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) power_3v3d = slave_result.power[1].result_3v3d, power_3v3a = slave_result.power[1].result_3v3a, power_n3va = slave_result.power[1].result_n3va, + is_slavelog_valid = true, runid, psbid, ), diff --git a/src/sql/create_table.sql b/src/sql/create_table.sql index 9331288..96b27d0 100644 --- a/src/sql/create_table.sql +++ b/src/sql/create_table.sql @@ -26,6 +26,8 @@ CREATE TABLE qaqc_single_run_results ( power_3v3a REAL, power_n3va REAL, note TEXT, + is_slavelog_valid BOOLEAN, + is_slaveclocklog_valid BOOLEAN, FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"), FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"), FOREIGN KEY("position") REFERENCES "qaqc_positions"("id") @@ -86,6 +88,7 @@ CREATE TABLE qaqc_extra_run_results ( invalid_register_value INTEGER, power_out_of_range INTEGER, note TEXT, + is_slavelog_valid BOOLEAN, FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"), FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"), FOREIGN KEY("position") REFERENCES "qaqc_positions"("id") diff --git a/test/runtests.jl b/test/runtests.jl index 049ce1c..ca409a6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -243,7 +243,7 @@ true || include("../src/PSBoardDataBase.jl") extra_100test_result_df = CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame) - @test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df) |> + @test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df, "input/slavelogs/") |> isnothing @test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |>