research/PSBoardDataBase
1
0
Fork 0
mirror of https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git synced 2025-06-08 05:55:42 +09:00
Code Issues Projects Releases Packages Wiki Activity Actions

[WIP] new(slavelog cache): save slavelog cache for extra run results to JLD2

Browse source
This commit is contained in:
Wataru Otsubo 2025-01-24 07:18:11 +01:00
parent cb898bd044
commit 733fb392c3
7 changed files with 160 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- `create_database_from_exported_csvs` now can save a JLD2 cache to store parsed slave logs.
## [0.5.1] - 2025-01-23
### Fixed

31
Manifest.toml
View file

@ -2,7 +2,7 @@
julia_version = "1.11.3"
manifest_format = "2.0"
project_hash = "e20af0781afd5866dc7398b8d852e9c8ee3e4f13"
project_hash = "47f0621871082a27225026b076304121300bf44d"
[[deps.ANSIColoredPrinters]]
git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"
@ -118,6 +118,18 @@ git-tree-sha1 = "e51db81749b0777b2147fbe7b783ee79045b8e99"
uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
version = "2.6.4+3"
[[deps.FileIO]]
deps = ["Pkg", "Requires", "UUIDs"]
git-tree-sha1 = "2dd20384bf8c6d411b5c7370865b1e9b26cb2ea3"
uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
version = "1.16.6"
[deps.FileIO.extensions]
HTTPExt = "HTTP"
[deps.FileIO.weakdeps]
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
[[deps.FilePathsBase]]
deps = ["Compat", "Dates"]
git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361"
@ -184,6 +196,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"
[[deps.JLD2]]
deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"]
git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.5.11"
[[deps.JLLWrappers]]
deps = ["Artifacts", "Preferences"]
git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6"
@ -250,6 +268,11 @@ version = "1.11.0"
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
version = "1.11.0"
[[deps.MacroTools]]
git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.15"
[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@ -370,6 +393,12 @@ git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51"
uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3"
version = "0.1.0"
[[deps.Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.3.0"
[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"

2
Project.toml
View file

@ -11,6 +11,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
@ -32,6 +33,7 @@ Dates = "1.10"
Documenter = "1.7"
Downloads = "1"
InteractiveUtils = "1.10"
JLD2 = "0.5.11"
Printf = "1.10"
SQLite = "1"
StaticArrays = "1.9"

33
docs/Manifest.toml
View file

@ -136,6 +136,18 @@ git-tree-sha1 = "e51db81749b0777b2147fbe7b783ee79045b8e99"
uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
version = "2.6.4+3"
[[deps.FileIO]]
deps = ["Pkg", "Requires", "UUIDs"]
git-tree-sha1 = "2dd20384bf8c6d411b5c7370865b1e9b26cb2ea3"
uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
version = "1.16.6"
[deps.FileIO.extensions]
HTTPExt = "HTTP"
[deps.FileIO.weakdeps]
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
[[deps.FilePathsBase]]
deps = ["Compat", "Dates"]
git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361"
@ -202,6 +214,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"
[[deps.JLD2]]
deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"]
git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.5.11"
[[deps.JLLWrappers]]
deps = ["Artifacts", "Preferences"]
git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6"
@ -268,6 +286,11 @@ version = "1.11.0"
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
version = "1.11.0"
[[deps.MacroTools]]
git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.15"
[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@ -324,10 +347,10 @@ uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15"
version = "10.42.0+1"
[[deps.PSBoardDataBase]]
deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "Printf", "SQLite", "StaticArrays", "Tables"]
deps = ["AutoHashEquals", "CSV", "DBInterface", "DataFrames", "Dates", "Documenter", "Downloads", "JLD2", "Printf", "SQLite", "StaticArrays", "Tables"]
path = "/home/qwjyh/Documents/school/lab/PSBoard_QAQC/PSBoardDataBase"
uuid = "779f6a9c-59fa-41f1-8ed1-e9a91eccb2f5"
version = "0.5.0"
version = "0.5.1"
weakdeps = ["InteractiveUtils"]
[deps.PSBoardDataBase.extensions]
@ -398,6 +421,12 @@ git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51"
uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3"
version = "0.1.0"
[[deps.Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.3.0"
[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"

33
src/PSBoardDataBase.jl
View file

@ -1,10 +1,12 @@
module PSBoardDataBase
using CSV: nothingerror
using SQLite
using DBInterface
using Tables
using CSV
using DataFrames
using JLD2
using Dates
include("QaqcMasterLog.jl")
@ -21,17 +23,20 @@ using .DispatchChecker
"""
create_database_from_exported_csvs(
dbpath::AbstractString;
masterlog_dir::AbstractString,
slavelog_dir::AbstractString,
slavelog_result::AbstractString,
single_run_csv::AbstractString = DownloadCSVs.download_single_run_csv(),
runlist_csv::AbstractString = DownloadCSVs.download_runlist_csv(),
dispatch_csv::AbstractString = DownloadCSVs.download_dispatch_csv(),
hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
masterlog_dir::AbstractString,
slavelog_dir::AbstractString,
)
Create database at `dbpath` and import data from CSV and master log files.
Optionally, you can make a cache file to store parsed slave logs.
# Arguments
## Required
@ -40,6 +45,12 @@ Create database at `dbpath` and import data from CSV and master log files.
- `slavelog_dir`: path to the directory where all JATHub slave logs are stored
## Optional
### Slave log cache
- `slavelog_result`: Filename of JLD2 where parsed slave logs are stored. If not specified, the data is not saved. See [JLD2 document](https://juliaio.github.io/JLD2.jl/stable/) for how to read/write JLD2 files.
### Source
By default, CSV files of Google Sheets are downloaded automatically.
If you want to use alternative CSV or are not online, you can specify the path of the CSVs manually.
- `single_run_csv`: CSV of single run results exported from the Google sheets database
- `runlist_csv`: CSV of run lists exported from the Google sheets database
- `dispatch_csv`: CSV of dispatch lists exported from the Google sheets database
@ -78,6 +89,16 @@ function create_database_from_exported_csvs(
end
end
end .|> first
jld2_slavelog_path = if haskey(kw, :slavelog_result)
kw[:slavelog_result]
else
nothing
end
jld2_slavelog = if isnothing(jld2_slavelog_path)
nothing
else
jldopen(jld2_slavelog_path, "w")
end
insert_version_info(db)
insert_qaqc_campaign_id(db)
@ -88,9 +109,13 @@ function create_database_from_exported_csvs(
add_qaqc_single_result(db, single_result_df, runlist_table)
add_qaqc_dispatch(db, dispatch_table)
add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir)
add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir, jld2_slavelog)
add_skew_from_slave_clk_logs(db, slavelog_dir)
add_slavelog_result(db, slavelog_dir)
add_slavelog_result(db, slavelog_dir, jld2_slavelog)
if !isnothing(jld2_slavelog)
close(jld2_slavelog)
end
db
end

68
src/import_data.jl
View file

@ -608,7 +608,12 @@ function get_num_tests_for_extra_runs(runid::Int64)
end
"""
add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) -> nothing
add_qaqc_100test_result(
db::SQLite.DB,
table::DataFrame,
logs_dir::AbstractString,
jld2_slavelog_path::Union{AbstractString, Nothing},
) -> nothing
Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
which is converted from a raw exported CSV.
@ -621,7 +626,12 @@ which is converted from a raw exported CSV.
# Detail
- skips psboards in `resistance_test_passed` with `passed == false`
"""
function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString)
function add_qaqc_100test_result(
db::SQLite.DB,
table::DataFrame,
logs_dir::AbstractString,
jld2_slavelog::Union{JLD2.JLDFile, Nothing},
)
position_id_map =
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
(i, s) = x
@ -696,6 +706,7 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
""",
)
lock_db = ReentrantLock()
lock_jld2 = ReentrantLock()
Threads.@threads for row in eachrow(table)
if ismissing(row.runid) || !(row.runid in qaqc_run_ids)
@ -706,18 +717,15 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
error("Runid $(row.runid) not found in `qaqc_runs` table.")
end
is_slavelog_valid = try
SlaveLogParser.parse_slavelog_file(
joinpath(
logs_dir,
"main",
"$(row.motherboard_id)_$(row.runid)_longrun.txt",
),
slavelog_filename = "$(row.motherboard_id)_$(row.runid)_longrun.txt"
is_slavelog_valid, slavelog_result = try
result = SlaveLogParser.parse_slavelog_file(
joinpath(logs_dir, "main", slavelog_filename),
)
true
true, result
catch e
@debug "Failed to parse slave log due to $(e)" catch_backtrace()
false
false, missing
end
lock(lock_db) do
@ -745,6 +753,16 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::Abst
),
)
end
lock(lock_jld2) do
if !isnothing(jld2_slavelog)
if haskey(jld2_slavelog, slavelog_filename)
@debug "slave log already included: $(slavelog_filename)"
else
jld2_slavelog[slavelog_filename] = slavelog_result
end
end
end
end
nothing
@ -830,12 +848,20 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
end
"""
add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
add_slavelog_result(
db::SQLite.DB,
logs_dir::AbstractString,
jld2_slavelog::Union{Nothing, JLD2.JLDFile},
)
Extract QAQC results from slave log files for single runs.
Slave log files are expected to located in certain format under `logs_dir`.
"""
function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
function add_slavelog_result(
db::SQLite.DB,
logs_dir::AbstractString,
jld2_slavelog::Union{Nothing, JLD2.JLDFile},
)
exclude_runs = (
(runid = 51, psbid = nothing, reason = "clock only"),
(runid = 175, psbid = nothing, reason = "broken files"),
@ -867,14 +893,14 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
""",
) |> Tables.columntable |> (tbl -> tbl.id)
slave_files =
slave_file_paths =
readdir(joinpath(logs_dir, "main"), join = true) |>
filter(contains(r"\d+_\d+\.txt"))
DBInterface.transaction(db) do
for file in slave_files
for file_path in slave_file_paths
psbid, runid, islongrun =
SlaveLogParser.get_psbid_runid_from_filename(basename(file))
SlaveLogParser.get_psbid_runid_from_filename(basename(file_path))
@assert !islongrun
# exclusion
@ -902,16 +928,16 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
if !(runid in runids)
@debug "runid: $(runid) not in run list (psbid: $(psbid)). Parsing slave log to test its format."
slave_result = SlaveLogParser.parse_slavelog_file(file)
slave_result = SlaveLogParser.parse_slavelog_file(file_path)
continue
end
# main
slave_result = try
SlaveLogParser.parse_slavelog_file(file)
SlaveLogParser.parse_slavelog_file(file_path)
catch e
throw(error("Failed to parse slave log file: $(file)\n$(e)"))
throw(error("Failed to parse slave log file: $(file_path)\n$(e)"))
end
@assert length(slave_result.power) == 1 "Too many power results for single run"
@ -927,6 +953,10 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
psbid,
),
)
if !isnothing(jld2_slavelog)
jld2_slavelog[basename(file_path)] = slave_result
end
end
end

19
test/runtests.jl
View file

@ -3,6 +3,7 @@ using PSBoardDataBase
using StaticArrays
using CSV, DataFrames
using SQLite, DBInterface
using JLD2
using Dates
# hack for LanguageServer
@ -200,8 +201,10 @@ true || include("../src/PSBoardDataBase.jl")
@testset "full integrated test" begin
dbpath = tempname()
jld2path = tempname()
db = PSBoardDataBase.create_database(dbpath)
@info "" db
jld2_slavelog = jldopen(jld2path, "w")
@info "" db jld2_slavelog
@test PSBoardDataBase.insert_version_info(db) |> isnothing
let stmt
@ -243,13 +246,21 @@ true || include("../src/PSBoardDataBase.jl")
extra_100test_result_df =
CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame)
@test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df, "input/slavelogs/") |>
isnothing
@test PSBoardDataBase.add_qaqc_100test_result(
db,
extra_100test_result_df,
"input/slavelogs/",
jld2_slavelog,
) |> isnothing
@test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |>
isnothing
@test PSBoardDataBase.add_slavelog_result(db, "input/slavelogs/") |> isnothing
@test PSBoardDataBase.add_slavelog_result(
db,
"input/slavelogs/",
jld2_slavelog,
) |> isnothing
run(`sqlitebrowser $dbpath`)