diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ac405..27069e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `create_database_from_exported_csvs` now can save a JLD2 cache to store parsed slave logs. + ## [0.5.1] - 2025-01-23 ### Fixed diff --git a/Manifest.toml b/Manifest.toml index 3379ebc..f00ba07 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.11.3" manifest_format = "2.0" -project_hash = "e20af0781afd5866dc7398b8d852e9c8ee3e4f13" +project_hash = "47f0621871082a27225026b076304121300bf44d" [[deps.ANSIColoredPrinters]] git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" @@ -118,6 +118,18 @@ git-tree-sha1 = "e51db81749b0777b2147fbe7b783ee79045b8e99" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" version = "2.6.4+3" +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "2dd20384bf8c6d411b5c7370865b1e9b26cb2ea3" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.16.6" + + [deps.FileIO.extensions] + HTTPExt = "HTTP" + + [deps.FileIO.weakdeps] + HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" + [[deps.FilePathsBase]] deps = ["Compat", "Dates"] git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361" @@ -184,6 +196,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" +[[deps.JLD2]] +deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] +git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279" +uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +version = "0.5.11" + [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" @@ -250,6 +268,11 @@ version = "1.11.0" uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" +[[deps.MacroTools]] +git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.15" + [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -370,6 +393,12 @@ git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51" uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3" version = "0.1.0" +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" diff --git a/Project.toml b/Project.toml index 8afe31a..9054215 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" @@ -32,6 +33,7 @@ Dates = "1.10" Documenter = "1.7" Downloads = "1" InteractiveUtils = "1.10" +JLD2 = "0.5.11" Printf = "1.10" SQLite = "1" StaticArrays = "1.9" diff --git a/docs/Manifest.toml b/docs/Manifest.toml index f94e5af..f47b747 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -136,6 +136,18 @@ git-tree-sha1 = "e51db81749b0777b2147fbe7b783ee79045b8e99" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" version = "2.6.4+3" +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "2dd20384bf8c6d411b5c7370865b1e9b26cb2ea3" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.16.6" + + [deps.FileIO.extensions] + HTTPExt = "HTTP" + + [deps.FileIO.weakdeps] + HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" + [[deps.FilePathsBase]] deps = ["Compat", "Dates"] git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361" @@ -202,6 +214,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" +[[deps.JLD2]] +deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] +git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279" +uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +version = "0.5.11" + [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" @@ -268,6 +286,11 @@ version = "1.11.0" uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" +[[deps.MacroTools]] +git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.15" + [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -324,10 +347,10 @@ uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" version = "10.42.0+1" [[deps.PSBoardDataBase]] -deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "Printf", "SQLite", "StaticArrays", "Tables"] +deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "JLD2", "Printf", "SQLite", "StaticArrays", "Tables"] path = "/home/qwjyh/Documents/school/lab/PSBoard_QAQC/PSBoardDataBase" uuid = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5" -version = "0.5.0" +version = "0.5.1" weakdeps = ["InteractiveUtils"] [deps.PSBoardDataBase.extensions] @@ -398,6 +421,12 @@ git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51" uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3" version = "0.1.0" +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" diff --git a/src/PSBoardDataBase.jl b/src/PSBoardDataBase.jl index 2d8e6da..e5056ba 100644 --- a/src/PSBoardDataBase.jl +++ b/src/PSBoardDataBase.jl @@ -1,10 +1,12 @@ module PSBoardDataBase +using CSV: nothingerror using SQLite using DBInterface using Tables using CSV using DataFrames +using JLD2 using Dates include("QaqcMasterLog.jl") @@ -21,17 +23,20 @@ using .DispatchChecker """ create_database_from_exported_csvs( dbpath::AbstractString; + masterlog_dir::AbstractString, + slavelog_dir::AbstractString, + slavelog_result::AbstractString, single_run_csv::AbstractString = DownloadCSVs.download_single_run_csv(), runlist_csv::AbstractString = DownloadCSVs.download_runlist_csv(), dispatch_csv::AbstractString = DownloadCSVs.download_dispatch_csv(), hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(), jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(), - masterlog_dir::AbstractString, - slavelog_dir::AbstractString, ) Create database at `dbpath` and import data from CSV and master log files. +Optionally, you can make a cache file to store parsed slave logs. + # Arguments ## Required @@ -40,6 +45,12 @@ Create database at `dbpath` and import data from CSV and master log files. - `slavelog_dir`: path to the directory where all JATHub slave logs are stored ## Optional +### Slave log cache +- `slavelog_result`: Filename of JLD2 where parsed slave logs are stored. If not specified, the data is not saved. See [JLD2 document](https://juliaio.github.io/JLD2.jl/stable/) for how to read/write JLD2 files. + +### Source +By default, CSV files of Google Sheets are downloaded automatically. +If you want to use alternative CSV or are not online, you can specify the path of the CSVs manually. - `single_run_csv`: CSV of single run results exported from the Google sheets database - `runlist_csv`: CSV of run lists exported from the Google sheets database - `dispatch_csv`: CSV of dispatch lists exported from the Google sheets database @@ -78,6 +89,16 @@ function create_database_from_exported_csvs( end end end .|> first + jld2_slavelog_path = if haskey(kw, :slavelog_result) + kw[:slavelog_result] + else + nothing + end + jld2_slavelog = if isnothing(jld2_slavelog_path) + nothing + else + jldopen(jld2_slavelog_path, "w") + end insert_version_info(db) insert_qaqc_campaign_id(db) @@ -88,9 +109,13 @@ function create_database_from_exported_csvs( add_qaqc_single_result(db, single_result_df, runlist_table) add_qaqc_dispatch(db, dispatch_table) add_qaqc_runlist_from_masterlogs(db, masterlog_dir) - add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir) + add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir, jld2_slavelog) add_skew_from_slave_clk_logs(db, slavelog_dir) - add_slavelog_result(db, slavelog_dir) + add_slavelog_result(db, slavelog_dir, jld2_slavelog) + + if !isnothing(jld2_slavelog) + close(jld2_slavelog) + end db end diff --git a/src/import_data.jl b/src/import_data.jl index d2f2609..f5b1eaa 100644 --- a/src/import_data.jl +++ b/src/import_data.jl @@ -608,7 +608,12 @@ function get_num_tests_for_extra_runs(runid::Int64) end """ - add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) -> nothing + add_qaqc_100test_result( + db::SQLite.DB, + table::DataFrame, + logs_dir::AbstractString, + jld2_slavelog_path::Union{AbstractString, Nothing}, + ) -> nothing Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame, which is converted from a raw exported CSV. @@ -621,7 +626,12 @@ which is converted from a raw exported CSV. # Detail - skips psboards in `resistance_test_passed` with `passed == false` """ -function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) +function add_qaqc_100test_result( + db::SQLite.DB, + table::DataFrame, + logs_dir::AbstractString, + jld2_slavelog::Union{JLD2.JLDFile, Nothing}, +) position_id_map = ["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin (i, s) = x @@ -696,6 +706,7 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst """, ) lock_db = ReentrantLock() + lock_jld2 = ReentrantLock() Threads.@threads for row in eachrow(table) if ismissing(row.runid) || !(row.runid in qaqc_run_ids) @@ -706,18 +717,15 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst error("Runid $(row.runid) not found in `qaqc_runs` table.") end - is_slavelog_valid = try - SlaveLogParser.parse_slavelog_file( - joinpath( - logs_dir, - "main", - "$(row.motherboard_id)_$(row.runid)_longrun.txt", - ), + slavelog_filename = "$(row.motherboard_id)_$(row.runid)_longrun.txt" + is_slavelog_valid, slavelog_result = try + result = SlaveLogParser.parse_slavelog_file( + joinpath(logs_dir, "main", slavelog_filename), ) - true + true, result catch e @debug "Failed to parse slave log due to $(e)" catch_backtrace() - false + false, missing end lock(lock_db) do @@ -745,6 +753,16 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst ), ) end + + lock(lock_jld2) do + if !isnothing(jld2_slavelog) + if haskey(jld2_slavelog, slavelog_filename) + @debug "slave log already included: $(slavelog_filename)" + else + jld2_slavelog[slavelog_filename] = slavelog_result + end + end + end end nothing @@ -830,12 +848,20 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) end """ - add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) + add_slavelog_result( + db::SQLite.DB, + logs_dir::AbstractString, + jld2_slavelog::Union{Nothing, JLD2.JLDFile}, + ) Extract QAQC results from slave log files for single runs. Slave log files are expected to located in certain format under `logs_dir`. """ -function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) +function add_slavelog_result( + db::SQLite.DB, + logs_dir::AbstractString, + jld2_slavelog::Union{Nothing, JLD2.JLDFile}, +) exclude_runs = ( (runid = 51, psbid = nothing, reason = "clock only"), (runid = 175, psbid = nothing, reason = "broken files"), @@ -867,14 +893,14 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) """, ) |> Tables.columntable |> (tbl -> tbl.id) - slave_files = + slave_file_paths = readdir(joinpath(logs_dir, "main"), join = true) |> filter(contains(r"\d+_\d+\.txt")) DBInterface.transaction(db) do - for file in slave_files + for file_path in slave_file_paths psbid, runid, islongrun = - SlaveLogParser.get_psbid_runid_from_filename(basename(file)) + SlaveLogParser.get_psbid_runid_from_filename(basename(file_path)) @assert !islongrun # exclusion @@ -902,16 +928,16 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) if !(runid in runids) @debug "runid: $(runid) not in run list (psbid: $(psbid)). Parsing slave log to test its format." - slave_result = SlaveLogParser.parse_slavelog_file(file) + slave_result = SlaveLogParser.parse_slavelog_file(file_path) continue end # main slave_result = try - SlaveLogParser.parse_slavelog_file(file) + SlaveLogParser.parse_slavelog_file(file_path) catch e - throw(error("Failed to parse slave log file: $(file)\n$(e)")) + throw(error("Failed to parse slave log file: $(file_path)\n$(e)")) end @assert length(slave_result.power) == 1 "Too many power results for single run" @@ -927,6 +953,10 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) psbid, ), ) + + if !isnothing(jld2_slavelog) + jld2_slavelog[basename(file_path)] = slave_result + end end end diff --git a/test/runtests.jl b/test/runtests.jl index ca409a6..1463f74 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,6 +3,7 @@ using PSBoardDataBase using StaticArrays using CSV, DataFrames using SQLite, DBInterface +using JLD2 using Dates # hack for LanguageServer @@ -200,8 +201,10 @@ true || include("../src/PSBoardDataBase.jl") @testset "full integrated test" begin dbpath = tempname() + jld2path = tempname() db = PSBoardDataBase.create_database(dbpath) - @info "" db + jld2_slavelog = jldopen(jld2path, "w") + @info "" db jld2_slavelog @test PSBoardDataBase.insert_version_info(db) |> isnothing let stmt @@ -243,13 +246,21 @@ true || include("../src/PSBoardDataBase.jl") extra_100test_result_df = CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame) - @test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df, "input/slavelogs/") |> - isnothing + @test PSBoardDataBase.add_qaqc_100test_result( + db, + extra_100test_result_df, + "input/slavelogs/", + jld2_slavelog, + ) |> isnothing @test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |> isnothing - @test PSBoardDataBase.add_slavelog_result(db, "input/slavelogs/") |> isnothing + @test PSBoardDataBase.add_slavelog_result( + db, + "input/slavelogs/", + jld2_slavelog, + ) |> isnothing run(`sqlitebrowser $dbpath`)