Merge branch 'add-slavelog-valid-flag-to-db' into 'main'

new(database column): add slavelog validity to single & extra run results

See merge request wotsubo/PSBoardDataBase!1418

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - QAQC progress plot notebook
 - (notebook) PLLLD wait time scan
 - Added QAQC campaign 7 to the database
+- Added slave log validity columns to single and extra run result tables (!1418)
 
 ### Changed
 

src/PSBoardDataBase.jl

@@ -88,7 +88,7 @@ function create_database_from_exported_csvs(
     add_qaqc_single_result(db, single_result_df, runlist_table)
     add_qaqc_dispatch(db, dispatch_table)
     add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
-    add_qaqc_100test_result(db, extra_100test_result_df)
+    add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir)
     add_skew_from_slave_clk_logs(db, slavelog_dir)
     add_slavelog_result(db, slavelog_dir)
 

src/import_data.jl

@@ -608,15 +608,20 @@ function get_num_tests_for_extra_runs(runid::Int64)
 end
 
 """
-    add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) -> nothing
+    add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) -> nothing
 
 Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
 which is converted from a raw exported CSV.
 
+# Args
+
+- `table`: 100 test result table, prepared with [`prepare_100test_table`](@ref)
+- `logs_dir`: where slave log files located in certain format
+
 # Detail
 - skips psboards in `resistance_test_passed` with `passed == false`
 """
-function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
+function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString)
     position_id_map =
         ["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
             (i, s) = x
@@ -625,22 +630,25 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
 
     table = prepare_100test_table(table)
 
-    stmt_search_runid = DBInterface.prepare(
+    qaqc_run_ids =
+        DBInterface.execute(
             db,
             sql"""
             SELECT id
             FROM qaqc_runs
-        WHERE id = :runid
             """,
-    )
-    stmt_search_resistance_error = DBInterface.prepare(
+        ) |> Tables.columntable |> (t -> t.id)
+    resistance_error_psb_list =
+        DBInterface.execute(
             db,
             sql"""
             SELECT psb_id
             FROM qaqc_resistance_check
-        WHERE psb_id = :psboard_id AND passed = 0
+            WHERE passed = 0
             """,
-    )
+        ) |>
+        Tables.columntable |>
+        (t -> t.psb_id)
     stmt_insert_result = DBInterface.prepare(
         db,
         sql"""
@@ -662,7 +670,8 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
                 bcid_fail,
                 invalid_register_value,
                 power_out_of_range,
-                note
+                note,
+                is_slavelog_valid
             )
         VALUES (
             :runid,
@@ -681,25 +690,37 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
             :bcid_fail,
             :invalid_register_value,
             :power_out_of_range,
-            :note
+            :note,
+            :is_slavelog_valid
         )
         """,
     )
+    lock_db = ReentrantLock()
 
-    for row in eachrow(table)
-        if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
+    Threads.@threads for row in eachrow(table)
+        if ismissing(row.runid) || !(row.runid in qaqc_run_ids)
             # search for resistance error
-            if !isempty(
-                DBInterface.execute(
-                    stmt_search_resistance_error,
-                    (; psboard_id = row.motherboard_id),
-                ),
-            )
+            if row.motherboard_id in resistance_error_psb_list
                 continue
             end
             error("Runid $(row.runid) not found in `qaqc_runs` table.")
         end
 
+        is_slavelog_valid = try
+            SlaveLogParser.parse_slavelog_file(
+                joinpath(
+                    logs_dir,
+                    "main",
+                    "$(row.motherboard_id)_$(row.runid)_longrun.txt",
+                ),
+            )
+            true
+        catch e
+            @debug "Failed to parse slave log due to $(e)" catch_backtrace()
+            false
+        end
+
+        lock(lock_db) do
             DBInterface.execute(
                 stmt_insert_result,
                 (
@@ -720,9 +741,11 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
                     invalid_register_value = row.var"invalid register values",
                     power_out_of_range = row.var"power out of range",
                     note = row.Column20,
+                    is_slavelog_valid,
                 ),
             )
         end
+    end
 
     nothing
 end
@@ -742,7 +765,7 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
         db,
         sql"""
         UPDATE qaqc_single_run_results
-        SET lvds_tx_skew = :skew
+        SET lvds_tx_skew = :skew, is_slaveclocklog_valid = :is_slaveclocklog_valid
         WHERE runid = :runid AND psboard_id = :psbid
         """,
     )
@@ -760,9 +783,27 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
 
             if m[:psbid] == "630" && m[:runid] == "190"
                 @debug "skipping... (psbid=630 runid=190 is broken)"
+                DBInterface.execute(
+                    stmt_insert_skew_to_single_result,
+                    (
+                        skew = missing,
+                        is_slaveclocklog_valid = false,
+                        runid = m[:runid],
+                        psbid = m[:psbid],
+                    ),
+                )
                 continue
             elseif m[:psbid] == "627" && m[:runid] == "344"
                 @debug "skipping... (psbid=627 runid=344 is broken)"
+                DBInterface.execute(
+                    stmt_insert_skew_to_single_result,
+                    (
+                        skew = missing,
+                        is_slaveclocklog_valid = false,
+                        runid = m[:runid],
+                        psbid = m[:psbid],
+                    ),
+                )
                 continue
             end
 
@@ -775,7 +816,12 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
 
             DBInterface.execute(
                 stmt_insert_skew_to_single_result,
-                (skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]),
+                (
+                    skew = ClockParser.get_skew(file),
+                    is_slaveclocklog_valid = true,
+                    runid = m[:runid],
+                    psbid = m[:psbid],
+                ),
             )
         end
     end
@@ -793,8 +839,8 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
     exclude_runs = (
         (runid = 51, psbid = nothing, reason = "clock only"),
         (runid = 175, psbid = nothing, reason = "broken files"),
-        (runid = 437, psbid = 1215, reason = "PSBID 1215 is not completed"),
-        (runid = 439, psbid = 703, reason = "PSBID 703 is not completed"),
+        (runid = 437, psbid = 1215, reason = "PSBID 1215 is not completed"), # debug 6.5
+        (runid = 439, psbid = 703, reason = "PSBID 703 is not completed"), # debug 6.5
         (runid = 434, psbid = 723, reason = "PSBID 723 is not completed"),
     )
     @assert eltype(exclude_runs) != Any
@@ -806,7 +852,8 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
         SET
             power_3v3d = :power_3v3d,
             power_3v3a = :power_3v3a,
-            power_n3va = :power_n3va
+            power_n3va = :power_n3va,
+            is_slavelog_valid = :is_slavelog_valid
         WHERE
             runid = :runid AND psboard_id = :psbid
         """,
@@ -839,6 +886,17 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
             end
             if !isempty(exclude_cond)
                 @debug "skipping runid = $(runid) for $(first(exclude_cond).reason)"
+                DBInterface.execute(
+                    stmt_insert_slave_result_to_single_result,
+                    (;
+                        power_3v3d = missing,
+                        power_3v3a = missing,
+                        power_n3va = missing,
+                        is_slavelog_valid = false,
+                        runid,
+                        psbid,
+                    ),
+                )
                 continue
             end
 
@@ -864,6 +922,7 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
                     power_3v3d = slave_result.power[1].result_3v3d,
                     power_3v3a = slave_result.power[1].result_3v3a,
                     power_n3va = slave_result.power[1].result_n3va,
+                    is_slavelog_valid = true,
                     runid,
                     psbid,
                 ),

src/sql/create_table.sql

@@ -26,6 +26,8 @@ CREATE TABLE qaqc_single_run_results (
   power_3v3a REAL,
   power_n3va REAL,
   note TEXT,
+  is_slavelog_valid BOOLEAN,
+  is_slaveclocklog_valid BOOLEAN,
   FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
   FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
   FOREIGN KEY("position") REFERENCES "qaqc_positions"("id")
@@ -86,6 +88,7 @@ CREATE TABLE qaqc_extra_run_results (
   invalid_register_value INTEGER,
   power_out_of_range INTEGER,
   note TEXT,
+  is_slavelog_valid BOOLEAN,
   FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
   FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
   FOREIGN KEY("position") REFERENCES "qaqc_positions"("id")

test/runtests.jl

@@ -243,7 +243,7 @@ true || include("../src/PSBoardDataBase.jl")
             extra_100test_result_df =
                 CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame)
 
-            @test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df) |>
+            @test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df, "input/slavelogs/") |>
                   isnothing
 
             @test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |>