[WIP] new(slavelog cache): save slavelog cache for extra run results to JLD2

src/PSBoardDataBase.jl

@@ -1,10 +1,12 @@
 module PSBoardDataBase
 
+using CSV: nothingerror
 using SQLite
 using DBInterface
 using Tables
 using CSV
 using DataFrames
+using JLD2
 using Dates
 
 include("QaqcMasterLog.jl")
@@ -21,17 +23,20 @@ using .DispatchChecker
 """
     create_database_from_exported_csvs(
         dbpath::AbstractString;
+        masterlog_dir::AbstractString,
+        slavelog_dir::AbstractString,
+        slavelog_result::AbstractString,
         single_run_csv::AbstractString = DownloadCSVs.download_single_run_csv(),
         runlist_csv::AbstractString = DownloadCSVs.download_runlist_csv(),
         dispatch_csv::AbstractString = DownloadCSVs.download_dispatch_csv(),
         hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
         jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
-        masterlog_dir::AbstractString,
-        slavelog_dir::AbstractString,
     )
 
 Create database at `dbpath` and import data from CSV and master log files.
 
+Optionally, you can make a cache file to store parsed slave logs.
+
 # Arguments
 
 ## Required
@@ -40,6 +45,12 @@ Create database at `dbpath` and import data from CSV and master log files.
 - `slavelog_dir`: path to the directory where all JATHub slave logs are stored
 
 ## Optional
+### Slave log cache
+- `slavelog_result`: Filename of JLD2 where parsed slave logs are stored. If not specified, the data is not saved. See [JLD2 document](https://juliaio.github.io/JLD2.jl/stable/) for how to read/write JLD2 files.
+
+### Source
+By default, CSV files of Google Sheets are downloaded automatically.
+If you want to use alternative CSV or are not online, you can specify the path of the CSVs manually.
 - `single_run_csv`: CSV of single run results exported from the Google sheets database
 - `runlist_csv`: CSV of run lists exported from the Google sheets database
 - `dispatch_csv`: CSV of dispatch lists exported from the Google sheets database
@@ -78,6 +89,16 @@ function create_database_from_exported_csvs(
                 end
             end
         end .|> first
+    jld2_slavelog_path = if haskey(kw, :slavelog_result)
+        kw[:slavelog_result]
+    else
+        nothing
+    end
+    jld2_slavelog = if isnothing(jld2_slavelog_path)
+        nothing
+    else
+        jldopen(jld2_slavelog_path, "w")
+    end
 
     insert_version_info(db)
     insert_qaqc_campaign_id(db)
@@ -88,9 +109,13 @@ function create_database_from_exported_csvs(
     add_qaqc_single_result(db, single_result_df, runlist_table)
     add_qaqc_dispatch(db, dispatch_table)
     add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
-    add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir)
+    add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir, jld2_slavelog)
     add_skew_from_slave_clk_logs(db, slavelog_dir)
-    add_slavelog_result(db, slavelog_dir)
+    add_slavelog_result(db, slavelog_dir, jld2_slavelog)
+
+    if !isnothing(jld2_slavelog)
+        close(jld2_slavelog)
+    end
 
     db
 end

src/import_data.jl

@@ -608,7 +608,12 @@ function get_num_tests_for_extra_runs(runid::Int64)
 end
 
 """
-    add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) -> nothing
+    add_qaqc_100test_result(
+        db::SQLite.DB,
+        table::DataFrame,
+        logs_dir::AbstractString,
+        jld2_slavelog_path::Union{AbstractString, Nothing},
+    ) -> nothing
 
 Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
 which is converted from a raw exported CSV.
@@ -621,7 +626,12 @@ which is converted from a raw exported CSV.
 # Detail
 - skips psboards in `resistance_test_passed` with `passed == false`
 """
-function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString)
+function add_qaqc_100test_result(
+    db::SQLite.DB,
+    table::DataFrame,
+    logs_dir::AbstractString,
+    jld2_slavelog::Union{JLD2.JLDFile, Nothing},
+)
     position_id_map =
         ["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
             (i, s) = x
@@ -696,6 +706,7 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
         """,
     )
     lock_db = ReentrantLock()
+    lock_jld2 = ReentrantLock()
 
     Threads.@threads for row in eachrow(table)
         if ismissing(row.runid) || !(row.runid in qaqc_run_ids)
@@ -706,18 +717,15 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
             error("Runid $(row.runid) not found in `qaqc_runs` table.")
         end
 
-        is_slavelog_valid = try
-            SlaveLogParser.parse_slavelog_file(
-                joinpath(
-                    logs_dir,
-                    "main",
-                    "$(row.motherboard_id)_$(row.runid)_longrun.txt",
-                ),
+        slavelog_filename = "$(row.motherboard_id)_$(row.runid)_longrun.txt"
+        is_slavelog_valid, slavelog_result = try
+            result = SlaveLogParser.parse_slavelog_file(
+                joinpath(logs_dir, "main", slavelog_filename),
             )
-            true
+            true, result
         catch e
             @debug "Failed to parse slave log due to $(e)" catch_backtrace()
-            false
+            false, missing
         end
 
         lock(lock_db) do
@@ -745,6 +753,16 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
                 ),
             )
         end
+
+        lock(lock_jld2) do
+            if !isnothing(jld2_slavelog)
+                if haskey(jld2_slavelog, slavelog_filename)
+                    @debug "slave log already included: $(slavelog_filename)"
+                else
+                    jld2_slavelog[slavelog_filename] = slavelog_result
+                end
+            end
+        end
     end
 
     nothing
@@ -830,12 +848,20 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
 end
 
 """
-    add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
+    add_slavelog_result(
+        db::SQLite.DB,
+        logs_dir::AbstractString,
+        jld2_slavelog::Union{Nothing, JLD2.JLDFile},
+    )
 
 Extract QAQC results from slave log files for single runs.
 Slave log files are expected to located in certain format under `logs_dir`.
 """
-function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
+function add_slavelog_result(
+    db::SQLite.DB,
+    logs_dir::AbstractString,
+    jld2_slavelog::Union{Nothing, JLD2.JLDFile},
+)
     exclude_runs = (
         (runid = 51, psbid = nothing, reason = "clock only"),
         (runid = 175, psbid = nothing, reason = "broken files"),
@@ -867,14 +893,14 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
             """,
         ) |> Tables.columntable |> (tbl -> tbl.id)
 
-    slave_files =
+    slave_file_paths =
         readdir(joinpath(logs_dir, "main"), join = true) |>
         filter(contains(r"\d+_\d+\.txt"))
 
     DBInterface.transaction(db) do
-        for file in slave_files
+        for file_path in slave_file_paths
             psbid, runid, islongrun =
-                SlaveLogParser.get_psbid_runid_from_filename(basename(file))
+                SlaveLogParser.get_psbid_runid_from_filename(basename(file_path))
             @assert !islongrun
 
             # exclusion
@@ -902,16 +928,16 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
 
             if !(runid in runids)
                 @debug "runid: $(runid) not in run list (psbid: $(psbid)). Parsing slave log to test its format."
-                slave_result = SlaveLogParser.parse_slavelog_file(file)
+                slave_result = SlaveLogParser.parse_slavelog_file(file_path)
                 continue
             end
 
             # main
 
             slave_result = try
-                SlaveLogParser.parse_slavelog_file(file)
+                SlaveLogParser.parse_slavelog_file(file_path)
             catch e
-                throw(error("Failed to parse slave log file: $(file)\n$(e)"))
+                throw(error("Failed to parse slave log file: $(file_path)\n$(e)"))
             end
 
             @assert length(slave_result.power) == 1 "Too many power results for single run"
@@ -927,6 +953,10 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
                     psbid,
                 ),
             )
+
+            if !isnothing(jld2_slavelog)
+                jld2_slavelog[basename(file_path)] = slave_result
+            end
         end
     end