[WIP] new(slavelog cache): save slavelog cache for extra run results to JLD2

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `create_database_from_exported_csvs` now can save a JLD2 cache to store parsed slave logs.
+
 ## [0.5.1] - 2025-01-23
 
 ### Fixed

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.11.3"
 manifest_format = "2.0"
-project_hash = "e20af0781afd5866dc7398b8d852e9c8ee3e4f13"
+project_hash = "47f0621871082a27225026b076304121300bf44d"
 
 [[deps.ANSIColoredPrinters]]
 git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"
@@ -118,6 +118,18 @@ git-tree-sha1 = "e51db81749b0777b2147fbe7b783ee79045b8e99"
 uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
 version = "2.6.4+3"
 
+[[deps.FileIO]]
+deps = ["Pkg", "Requires", "UUIDs"]
+git-tree-sha1 = "2dd20384bf8c6d411b5c7370865b1e9b26cb2ea3"
+uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+version = "1.16.6"
+
+    [deps.FileIO.extensions]
+    HTTPExt = "HTTP"
+
+    [deps.FileIO.weakdeps]
+    HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+
 [[deps.FilePathsBase]]
 deps = ["Compat", "Dates"]
 git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361"
@@ -184,6 +196,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
 uuid = "82899510-4779-5014-852e-03e436cf321d"
 version = "1.0.0"
 
+[[deps.JLD2]]
+deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"]
+git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279"
+uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+version = "0.5.11"
+
 [[deps.JLLWrappers]]
 deps = ["Artifacts", "Preferences"]
 git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6"
@@ -250,6 +268,11 @@ version = "1.11.0"
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 version = "1.11.0"
 
+[[deps.MacroTools]]
+git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.15"
+
 [[deps.Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@@ -370,6 +393,12 @@ git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51"
 uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3"
 version = "0.1.0"
 
+[[deps.Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.3.0"
+
 [[deps.SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 version = "0.7.0"

Project.toml

@@ -11,6 +11,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
@@ -32,6 +33,7 @@ Dates = "1.10"
 Documenter = "1.7"
 Downloads = "1"
 InteractiveUtils = "1.10"
+JLD2 = "0.5.11"
 Printf = "1.10"
 SQLite = "1"
 StaticArrays = "1.9"

docs/Manifest.toml

@@ -136,6 +136,18 @@ git-tree-sha1 = "e51db81749b0777b2147fbe7b783ee79045b8e99"
 uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
 version = "2.6.4+3"
 
+[[deps.FileIO]]
+deps = ["Pkg", "Requires", "UUIDs"]
+git-tree-sha1 = "2dd20384bf8c6d411b5c7370865b1e9b26cb2ea3"
+uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+version = "1.16.6"
+
+    [deps.FileIO.extensions]
+    HTTPExt = "HTTP"
+
+    [deps.FileIO.weakdeps]
+    HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+
 [[deps.FilePathsBase]]
 deps = ["Compat", "Dates"]
 git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361"
@@ -202,6 +214,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
 uuid = "82899510-4779-5014-852e-03e436cf321d"
 version = "1.0.0"
 
+[[deps.JLD2]]
+deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"]
+git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279"
+uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+version = "0.5.11"
+
 [[deps.JLLWrappers]]
 deps = ["Artifacts", "Preferences"]
 git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6"
@@ -268,6 +286,11 @@ version = "1.11.0"
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 version = "1.11.0"
 
+[[deps.MacroTools]]
+git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.15"
+
 [[deps.Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@@ -324,10 +347,10 @@ uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15"
 version = "10.42.0+1"
 
 [[deps.PSBoardDataBase]]
-deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "Printf", "SQLite", "StaticArrays", "Tables"]
+deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "JLD2", "Printf", "SQLite", "StaticArrays", "Tables"]
 path = "/home/qwjyh/Documents/school/lab/PSBoard_QAQC/PSBoardDataBase"
 uuid = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5"
-version = "0.5.0"
+version = "0.5.1"
 weakdeps = ["InteractiveUtils"]
 
     [deps.PSBoardDataBase.extensions]
@@ -398,6 +421,12 @@ git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51"
 uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3"
 version = "0.1.0"
 
+[[deps.Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.3.0"
+
 [[deps.SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 version = "0.7.0"

src/PSBoardDataBase.jl

@@ -1,10 +1,12 @@
 module PSBoardDataBase
 
+using CSV: nothingerror
 using SQLite
 using DBInterface
 using Tables
 using CSV
 using DataFrames
+using JLD2
 using Dates
 
 include("QaqcMasterLog.jl")
@@ -21,17 +23,20 @@ using .DispatchChecker
 """
     create_database_from_exported_csvs(
         dbpath::AbstractString;
+        masterlog_dir::AbstractString,
+        slavelog_dir::AbstractString,
+        slavelog_result::AbstractString,
         single_run_csv::AbstractString = DownloadCSVs.download_single_run_csv(),
         runlist_csv::AbstractString = DownloadCSVs.download_runlist_csv(),
         dispatch_csv::AbstractString = DownloadCSVs.download_dispatch_csv(),
         hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
         jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
-        masterlog_dir::AbstractString,
-        slavelog_dir::AbstractString,
     )
 
 Create database at `dbpath` and import data from CSV and master log files.
 
+Optionally, you can make a cache file to store parsed slave logs.
+
 # Arguments
 
 ## Required
@@ -40,6 +45,12 @@ Create database at `dbpath` and import data from CSV and master log files.
 - `slavelog_dir`: path to the directory where all JATHub slave logs are stored
 
 ## Optional
+### Slave log cache
+- `slavelog_result`: Filename of JLD2 where parsed slave logs are stored. If not specified, the data is not saved. See [JLD2 document](https://juliaio.github.io/JLD2.jl/stable/) for how to read/write JLD2 files.
+
+### Source
+By default, CSV files of Google Sheets are downloaded automatically.
+If you want to use alternative CSV or are not online, you can specify the path of the CSVs manually.
 - `single_run_csv`: CSV of single run results exported from the Google sheets database
 - `runlist_csv`: CSV of run lists exported from the Google sheets database
 - `dispatch_csv`: CSV of dispatch lists exported from the Google sheets database
@@ -78,6 +89,16 @@ function create_database_from_exported_csvs(
                 end
             end
         end .|> first
+    jld2_slavelog_path = if haskey(kw, :slavelog_result)
+        kw[:slavelog_result]
+    else
+        nothing
+    end
+    jld2_slavelog = if isnothing(jld2_slavelog_path)
+        nothing
+    else
+        jldopen(jld2_slavelog_path, "w")
+    end
 
     insert_version_info(db)
     insert_qaqc_campaign_id(db)
@@ -88,9 +109,13 @@ function create_database_from_exported_csvs(
     add_qaqc_single_result(db, single_result_df, runlist_table)
     add_qaqc_dispatch(db, dispatch_table)
     add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
-    add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir)
+    add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir, jld2_slavelog)
     add_skew_from_slave_clk_logs(db, slavelog_dir)
-    add_slavelog_result(db, slavelog_dir)
+    add_slavelog_result(db, slavelog_dir, jld2_slavelog)
+
+    if !isnothing(jld2_slavelog)
+        close(jld2_slavelog)
+    end
 
     db
 end

src/import_data.jl

@@ -608,7 +608,12 @@ function get_num_tests_for_extra_runs(runid::Int64)
 end
 
 """
-    add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) -> nothing
+    add_qaqc_100test_result(
+        db::SQLite.DB,
+        table::DataFrame,
+        logs_dir::AbstractString,
+        jld2_slavelog_path::Union{AbstractString, Nothing},
+    ) -> nothing
 
 Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
 which is converted from a raw exported CSV.
@@ -621,7 +626,12 @@ which is converted from a raw exported CSV.
 # Detail
 - skips psboards in `resistance_test_passed` with `passed == false`
 """
-function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString)
+function add_qaqc_100test_result(
+    db::SQLite.DB,
+    table::DataFrame,
+    logs_dir::AbstractString,
+    jld2_slavelog::Union{JLD2.JLDFile, Nothing},
+)
     position_id_map =
         ["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
             (i, s) = x
@@ -696,6 +706,7 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
         """,
     )
     lock_db = ReentrantLock()
+    lock_jld2 = ReentrantLock()
 
     Threads.@threads for row in eachrow(table)
         if ismissing(row.runid) || !(row.runid in qaqc_run_ids)
@@ -706,18 +717,15 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
             error("Runid $(row.runid) not found in `qaqc_runs` table.")
         end
 
-        is_slavelog_valid = try
+        slavelog_filename = "$(row.motherboard_id)_$(row.runid)_longrun.txt"
-            SlaveLogParser.parse_slavelog_file(
+        is_slavelog_valid, slavelog_result = try
-                joinpath(
+            result = SlaveLogParser.parse_slavelog_file(
-                    logs_dir,
+                joinpath(logs_dir, "main", slavelog_filename),
-                    "main",
-                    "$(row.motherboard_id)_$(row.runid)_longrun.txt",
-                ),
             )
-            true
+            true, result
         catch e
             @debug "Failed to parse slave log due to $(e)" catch_backtrace()
-            false
+            false, missing
         end
 
         lock(lock_db) do
@@ -745,6 +753,16 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
                 ),
             )
         end
+
+        lock(lock_jld2) do
+            if !isnothing(jld2_slavelog)
+                if haskey(jld2_slavelog, slavelog_filename)
+                    @debug "slave log already included: $(slavelog_filename)"
+                else
+                    jld2_slavelog[slavelog_filename] = slavelog_result
+                end
+            end
+        end
     end
 
     nothing
@@ -830,12 +848,20 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
 end
 
 """
-    add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
+    add_slavelog_result(
+        db::SQLite.DB,
+        logs_dir::AbstractString,
+        jld2_slavelog::Union{Nothing, JLD2.JLDFile},
+    )
 
 Extract QAQC results from slave log files for single runs.
 Slave log files are expected to located in certain format under `logs_dir`.
 """
-function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
+function add_slavelog_result(
+    db::SQLite.DB,
+    logs_dir::AbstractString,
+    jld2_slavelog::Union{Nothing, JLD2.JLDFile},
+)
     exclude_runs = (
         (runid = 51, psbid = nothing, reason = "clock only"),
         (runid = 175, psbid = nothing, reason = "broken files"),
@@ -867,14 +893,14 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
             """,
         ) |> Tables.columntable |> (tbl -> tbl.id)
 
-    slave_files =
+    slave_file_paths =
         readdir(joinpath(logs_dir, "main"), join = true) |>
         filter(contains(r"\d+_\d+\.txt"))
 
     DBInterface.transaction(db) do
-        for file in slave_files
+        for file_path in slave_file_paths
             psbid, runid, islongrun =
-                SlaveLogParser.get_psbid_runid_from_filename(basename(file))
+                SlaveLogParser.get_psbid_runid_from_filename(basename(file_path))
             @assert !islongrun
 
             # exclusion
@@ -902,16 +928,16 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
 
             if !(runid in runids)
                 @debug "runid: $(runid) not in run list (psbid: $(psbid)). Parsing slave log to test its format."
-                slave_result = SlaveLogParser.parse_slavelog_file(file)
+                slave_result = SlaveLogParser.parse_slavelog_file(file_path)
                 continue
             end
 
             # main
 
             slave_result = try
-                SlaveLogParser.parse_slavelog_file(file)
+                SlaveLogParser.parse_slavelog_file(file_path)
             catch e
-                throw(error("Failed to parse slave log file: $(file)\n$(e)"))
+                throw(error("Failed to parse slave log file: $(file_path)\n$(e)"))
             end
 
             @assert length(slave_result.power) == 1 "Too many power results for single run"
@@ -927,6 +953,10 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
                     psbid,
                 ),
             )
+
+            if !isnothing(jld2_slavelog)
+                jld2_slavelog[basename(file_path)] = slave_result
+            end
         end
     end
 

test/runtests.jl

@@ -3,6 +3,7 @@ using PSBoardDataBase
 using StaticArrays
 using CSV, DataFrames
 using SQLite, DBInterface
+using JLD2
 using Dates
 
 # hack for LanguageServer
@@ -200,8 +201,10 @@ true || include("../src/PSBoardDataBase.jl")
 
     @testset "full integrated test" begin
         dbpath = tempname()
+        jld2path = tempname()
         db = PSBoardDataBase.create_database(dbpath)
-        @info "" db
+        jld2_slavelog = jldopen(jld2path, "w")
+        @info "" db jld2_slavelog
 
         @test PSBoardDataBase.insert_version_info(db) |> isnothing
         let stmt
@@ -243,13 +246,21 @@ true || include("../src/PSBoardDataBase.jl")
             extra_100test_result_df =
                 CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame)
 
-            @test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df, "input/slavelogs/") |>
+            @test PSBoardDataBase.add_qaqc_100test_result(
-                  isnothing
+                db,
+                extra_100test_result_df,
+                "input/slavelogs/",
+                jld2_slavelog,
+            ) |> isnothing
 
             @test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |>
                   isnothing
 
-            @test PSBoardDataBase.add_slavelog_result(db, "input/slavelogs/") |> isnothing
+            @test PSBoardDataBase.add_slavelog_result(
+                db,
+                "input/slavelogs/",
+                jld2_slavelog,
+            ) |> isnothing
 
             run(`sqlitebrowser $dbpath`)