[new] Slave clock parser and function to insert into database

src/PSBoardDataBase.jl

@@ -8,6 +8,7 @@ using DataFrames
 using Dates
 
 include("parse_qaqc_master_log.jl")
+include("parse_clock.jl")
 
 include("create_table.jl")
 include("download_csv.jl")
@@ -22,6 +23,7 @@ include("import_data.jl")
         hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
         jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
         masterlog_dir::AbstractString,
+        slavelog_dir::AbstractString,
     )
 
 Create database at `dbpath` and import data from CSV and master log files.
@@ -33,7 +35,8 @@ Create database at `dbpath` and import data from CSV and master log files.
 - `dispatch_csv`: CSV of dispatch lists exported from the Google sheets database
 - `hundred_csv`: CSV of 100 tests results exported from the Google sheets database
 - `jathubs_csv`: CSV for jathub list used in QAQC. Used to add skew.
-- `masterlog_dir`: path to the directory (`log`) where all JATHub master log is stored
+- `masterlog_dir`: path to the directory (`log`) where all JATHub master logs are stored
+- `slavelog_dir`: path to the directory where all JATHub slave logs are stored
 """
 function create_database_from_exported_csvs(
     dbpath::AbstractString;
@@ -43,6 +46,7 @@ function create_database_from_exported_csvs(
     hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
     jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
     masterlog_dir::AbstractString,
+    slavelog_dir::AbstractString,
 )
     db = create_database(dbpath)
     single_result_df = CSV.read(single_run_csv, DataFrame)
@@ -59,6 +63,7 @@ function create_database_from_exported_csvs(
     add_qaqc_dispatch(db, dispatch_table)
     add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
     add_qaqc_100test_result(db, extra_100test_result_df)
+    add_skew_from_slave_clk_logs(db, slavelog_dir)
 
     db
 end

src/import_data.jl

@@ -702,3 +702,43 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
 
     nothing
 end
+
+"""
+    add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
+
+Insert skew measurement result from slave logs with name `psbid_runid_clk.txt`.
+See [`ClockParser.get_skew`](@ref) for parse detail.
+
+# Abnormal logs:
+- `48_nagoya_irradition_...`: skipped
+"""
+function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
+    stmt_insrt = DBInterface.prepare(
+        db,
+        sql"""
+        UPDATE qaqc_single_run_results
+        SET lvds_tx_skew = :skew
+        WHERE runid = :runid AND psboard_id = :psbid
+        """,
+    )
+    clk_files =
+        readdir("$logs_dir/main", join = true) |>
+        filter(endswith("_clk.txt")) |>
+        filter(!contains("nagoya"))
+
+    DBInterface.transaction(db) do
+        for file in clk_files
+            m = match(r"^(?<psbid>\d+)_(?<runid>\d+)_clk.txt$", splitdir(file) |> last)
+            if isnothing(m)
+                error("Invalid filename $(file)")
+            end
+
+            DBInterface.execute(
+                stmt_insrt,
+                (skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]),
+            )
+        end
+    end
+
+    nothing
+end

src/parse_clock.jl

@@ -0,0 +1,46 @@
+module ClockParser
+
+export get_skew
+
+function _parse_line(line::AbstractString)
+    time, high, _ = split(line)
+    parse(Float64, time), parse(Float64, high)
+end
+
+"""
+    get_skew(file::T) where {T <: AbstractString} -> Union{Float64, Missing}
+
+Get skew from clock result file `file`.
+It returns `missing` for invalid files.
+To see the detailed reason, increase log level to `DEBUG`.
+Invalid cases are:
+
+- first line has >0 counts => "Unexpected first line"
+- no measurement has >500 counts => "Clock skew out of range"
+"""
+function get_skew(file::T) where {T <: AbstractString}
+    lines = Iterators.Stateful(eachline(file))
+
+    first_line = popfirst!(lines)
+    # criteria: https://gitlab.cern.ch/dhashimo/PS_Board_QAQC/-/blob/master/Software_Test/project_auto_reset/QAQC_test_func.cxx?ref_type=heads#L208
+    let
+        time, high = _parse_line(first_line)
+        if high > 0
+            @debug "Unexpected first line" file
+            return missing
+        elseif high > 500
+            return time
+        end
+    end
+
+    time_and_counts = Iterators.map(_parse_line, lines)
+    for (time, high) in time_and_counts
+        if high > 500
+            return time
+        end
+    end
+    @debug "Clock skew out of range"
+    return missing
+end
+
+end # module ClockParser

src/sql/create_table.sql

@@ -21,6 +21,7 @@ CREATE TABLE qaqc_single_run_results (
   asdtp INTEGER,
   reset INTEGER,
   qaqc_result INTEGER,
+  lvds_tx_skew REAL,
   note TEXT,
   FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
   FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
@@ -114,8 +115,8 @@ AS
     qaqc_single_run_results.asdtp,
     qaqc_single_run_results.reset,
     qaqc_single_run_results.qaqc_result,
+    qaqc_single_run_results.lvds_tx_skew - qaqc_positions.rising_ns AS lvds_tx_skew,
     qaqc_runs.shiftscript_ver,
-    qaqc_runs.shifter,
     qaqc_single_run_results.note AS result_note
   FROM
     ps_boards,