new(create_database_from_exported_csvs): integrate slave log temp result

- breaking: add three columns(power_3v3d, power_3v3a, power_n3va) to single_result_table
- modified sql statements in functions in import_data.jl to have more meaning
- integrate to main `create_database_from_exported_csvs` func

src/PSBoardDataBase.jl

@@ -73,6 +73,7 @@ function create_database_from_exported_csvs(
     add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
     add_qaqc_100test_result(db, extra_100test_result_df)
     add_skew_from_slave_clk_logs(db, slavelog_dir)
+    add_slavelog_result(db, slavelog_dir)
 
     db
 end

src/SlaveLogParser.jl

@@ -33,6 +33,8 @@ struct SlaveLogResult end
     get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64,Int64,Bool}
 
 Extract info from slave log filename.
+
+Returns a tuple of `psbid`, `runid`, `islongrun`.
 """
 function get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64, Int64, Bool}
     main, _ext = splitext(filename)

src/import_data.jl

@@ -4,13 +4,16 @@
 Insert version information of this software as string.
 """
 function insert_version_info(db::SQLite.DB)
-    stmt = DBInterface.prepare(
+    stmt_insert_version = DBInterface.prepare(
         db,
         sql"""
         INSERT INTO versions VALUES (:converter)
         """,
     )
-    DBInterface.execute(stmt, (; converter = pkgversion(@__MODULE__) |> string))
+    DBInterface.execute(
+        stmt_insert_version,
+        (; converter = pkgversion(@__MODULE__) |> string),
+    )
 
     nothing
 end
@@ -28,12 +31,12 @@ function insert_qaqc_campaign_id(db::SQLite.DB)
         (DateTime(2024, 9, 10), DateTime(2024, 9, 12)),
         (DateTime(2024, 9, 30), DateTime(2024, 10, 4)),
     ]
-    stmt = DBInterface.prepare(
+    stmt_insert_campaigns = DBInterface.prepare(
         db,
         sql"INSERT INTO qaqc_campaigns VALUES (:id, :start_date, :end_date, :note)",
     )
     DBInterface.executemany(
-        stmt,
+        stmt_insert_campaigns,
         (
             id = campaigns,
             start_date = dates .|> (x -> x[1]) .|> string,
@@ -58,7 +61,7 @@ function insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame)
         Symbol("立ち上がり [ns]") => ByRow(Float64) => Symbol("立ち上がり [ns]"),
     )
 
-    stmt = DBInterface.prepare(
+    stmt_insert_positions = DBInterface.prepare(
         db,
         sql"""
         INSERT INTO qaqc_positions
@@ -72,7 +75,7 @@ function insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame)
         """,
     )
     DBInterface.executemany(
-        stmt,
+        stmt_insert_positions,
         (
             id = 1:18,
             name = ["B-$i-$j" for i in 0:1 for j in 1:9],
@@ -138,12 +141,12 @@ function add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame)
     end
     filter!(:motherboard_id => !=(999999), df)
 
-    stmt = DBInterface.prepare(
+    stmt_insert_psbid = DBInterface.prepare(
         db,
         sql"INSERT INTO ps_boards VALUES (:psbid, :daughterboardid)",
     )
     DBInterface.executemany(
-        stmt,
+        stmt_insert_psbid,
         (psbid = df.motherboard_id, daughterboardid = df.daughterboard),
     )
 
@@ -426,7 +429,7 @@ function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
     dispatch_table = prepare_dispatch_table(dispatch_table)
 
     # TODO: provide datetime
-    stmt = DBInterface.prepare(
+    stmt_insert_dispatch = DBInterface.prepare(
         db,
         sql"""
         INSERT INTO qaqc_dispatch(qaqc_campaign_id, psb_id, source_place, destination, time)
@@ -435,7 +438,7 @@ function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
     )
 
     DBInterface.executemany(
-        stmt,
+        stmt_insert_dispatch,
         (campaign_id = dispatch_table.campaign_id, psboard_id = dispatch_table.psboard_id),
     )
 
@@ -717,7 +720,7 @@ See [`ClockParser.get_skew`](@ref) for parse detail.
 - `630_190`: broken file
 """
 function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
-    stmt_insrt = DBInterface.prepare(
+    stmt_insert_skew_to_single_result = DBInterface.prepare(
         db,
         sql"""
         UPDATE qaqc_single_run_results
@@ -726,7 +729,7 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
         """,
     )
     clk_files =
-        readdir("$logs_dir/main", join = true) |>
+        readdir(joinpath(logs_dir, "main"), join = true) |>
         filter(endswith("_clk.txt")) |>
         filter(!contains("nagoya"))
 
@@ -743,7 +746,7 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
             end
 
             DBInterface.execute(
-                stmt_insrt,
+                stmt_insert_skew_to_single_result,
                 (skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]),
             )
         end
@@ -751,3 +754,82 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
 
     nothing
 end
+
+"""
+    add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
+
+Extract QAQC results from slave log files for single runs.
+Slave log files are expected to located in certain format under `logs_dir`.
+"""
+function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
+    exclude_runs =
+        ((runid = 51, reason = "clock only"), (runid = 175, reason = "broken files"))
+
+    stmt_insert_slave_result_to_single_result = DBInterface.prepare(
+        db,
+        sql"""
+        UPDATE qaqc_single_run_results
+        SET
+            power_3v3d = :power_3v3d,
+            power_3v3a = :power_3v3a,
+            power_n3va = :power_n3va
+        WHERE
+            runid = :runid AND psboard_id = :psbid
+        """,
+    )
+    runids =
+        DBInterface.execute(
+            db,
+            sql"""
+            SELECT id
+            FROM qaqc_runs
+            """,
+        ) |> Tables.columntable |> (tbl -> tbl.id)
+
+    slave_files =
+        readdir(joinpath(logs_dir, "main"), join = true) |>
+        filter(contains(r"\d+_\d+\.txt"))
+
+    DBInterface.transaction(db) do
+        for file in slave_files
+            psbid, runid, islongrun =
+                SlaveLogParser.get_psbid_runid_from_filename(basename(file))
+            @assert !islongrun
+
+            # exclusion
+
+            exclude_cond = Iterators.filter(exclude_runs) do cond
+                runid == cond.runid
+            end
+            if !isempty(exclude_cond)
+                @debug "skipping runid = $(runid) for $(first(exclude_cond).reason)"
+                continue
+            end
+
+            if !(runid in runids)
+                slave_result = SlaveLogParser.parse_slavelog_file(file)
+                @debug "runid: $(runid) not in run list (psbid: $(psbid))"
+                continue
+            end
+
+            # main
+
+            slave_result = SlaveLogParser.parse_slavelog_file(file)
+
+            @assert length(slave_result.power) == 1 "Too many power results for single run"
+
+            DBInterface.execute(
+                stmt_insert_slave_result_to_single_result,
+                (;
+                    power_3v3d = slave_result.power[1].result_3v3d,
+                    power_3v3a = slave_result.power[1].result_3v3a,
+                    power_n3va = slave_result.power[1].result_n3va,
+                    runid,
+                    psbid,
+                ),
+            )
+        end
+    end
+
+    nothing
+end

src/parse_clock.jl

@@ -19,7 +19,7 @@ Invalid cases are:
 - no measurement has >500 counts => "Clock skew out of range"
 """
 function get_skew(file::T) where {T <: AbstractString}
-    @debug "file: $(file)"
+    # @debug "file: $(file)"
     lines = Iterators.Stateful(eachline(file))
 
     was_0_before = false
@@ -40,7 +40,7 @@ function get_skew(file::T) where {T <: AbstractString}
             return time
         end
     end
-    @debug "Clock skew out of range"
+    @debug "Clock skew out of range (file: $(file))"
     return missing
 end
 

src/sql/create_table.sql

@@ -22,6 +22,9 @@ CREATE TABLE qaqc_single_run_results (
   reset INTEGER,
   qaqc_result INTEGER,
   lvds_tx_skew REAL,
+  power_3v3d REAL,
+  power_3v3a REAL,
+  power_n3va REAL,
   note TEXT,
   FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
   FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),

test/input/.gitignore

@@ -9,3 +9,4 @@ slavelogs/main/*
 !slavelogs/main/525_245_longrun.txt
 !slavelogs/main/430_100.txt
 !slavelogs/main/364_88_longrun.txt
+!slavelogs/main/127_172.txt

test/runtests.jl

@@ -97,16 +97,25 @@ true || include("../src/PSBoardDataBase.jl")
             @test result.result_3v3d == 3.47
             @test result.result_3v3a == 2.91
             @test result.result_n3va == -3.01
+
+            lines = Iterators.Stateful(
+                Iterators.drop(eachline("./input/slavelogs/main/127_172.txt"), 17),
+            )
+            result = PSBoardDataBase.SlaveLogParser.parse_power_section!(lines)
+            @test result.result_3v3d == 7.65
+            @test result.result_3v3a == 3.80
+            @test result.result_n3va == -2.87
+            @test result.fpga_temp == -63.21
         end
 
         @testset "Recov" begin
             lines = Iterators.Stateful(
-                Iterators.drop(eachline("./input/slavelogs/main/430_100.txt"), 1912)
+                Iterators.drop(eachline("./input/slavelogs/main/430_100.txt"), 1912),
             )
             @test PSBoardDataBase.SlaveLogParser.parse_recov_section!(lines)
 
             lines = Iterators.Stateful(
-                Iterators.drop(eachline("./input/slavelogs/main/525_244.txt"), 1912)
+                Iterators.drop(eachline("./input/slavelogs/main/525_244.txt"), 1912),
             )
             @test PSBoardDataBase.SlaveLogParser.parse_recov_section!(lines)
 
@@ -240,6 +249,8 @@ true || include("../src/PSBoardDataBase.jl")
             @test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |>
                   isnothing
 
+            @test PSBoardDataBase.add_slavelog_result(db, "input/slavelogs/") |> isnothing
+
             run(`sqlitebrowser $dbpath`)
 
             @test PSBoardDataBase.create_database_from_exported_csvs(