From dda24bc68a87b7757da9fb0993ea7a8b1cb25f6a Mon Sep 17 00:00:00 2001 From: Wataru Otsubo Date: Fri, 1 Nov 2024 23:27:58 +0900 Subject: [PATCH] new(create_database_from_exported_csvs): integrate slave log temp result - breaking: add three columns(power_3v3d, power_3v3a, power_n3va) to single_result_table - modified sql statements in functions in import_data.jl to have more meaning - integrate to main `create_database_from_exported_csvs` func --- src/PSBoardDataBase.jl | 1 + src/SlaveLogParser.jl | 2 + src/import_data.jl | 108 ++++++++++++++++++++++++++++++++++----- src/parse_clock.jl | 4 +- src/sql/create_table.sql | 3 ++ test/input/.gitignore | 1 + test/runtests.jl | 15 +++++- 7 files changed, 117 insertions(+), 17 deletions(-) diff --git a/src/PSBoardDataBase.jl b/src/PSBoardDataBase.jl index 1ab8b2b..5eda62c 100644 --- a/src/PSBoardDataBase.jl +++ b/src/PSBoardDataBase.jl @@ -73,6 +73,7 @@ function create_database_from_exported_csvs( add_qaqc_runlist_from_masterlogs(db, masterlog_dir) add_qaqc_100test_result(db, extra_100test_result_df) add_skew_from_slave_clk_logs(db, slavelog_dir) + add_slavelog_result(db, slavelog_dir) db end diff --git a/src/SlaveLogParser.jl b/src/SlaveLogParser.jl index f2ad3b5..85fb8fe 100644 --- a/src/SlaveLogParser.jl +++ b/src/SlaveLogParser.jl @@ -33,6 +33,8 @@ struct SlaveLogResult end get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64,Int64,Bool} Extract info from slave log filename. + +Returns a tuple of `psbid`, `runid`, `islongrun`. """ function get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64, Int64, Bool} main, _ext = splitext(filename) diff --git a/src/import_data.jl b/src/import_data.jl index e3647c3..29e7aaa 100644 --- a/src/import_data.jl +++ b/src/import_data.jl @@ -4,13 +4,16 @@ Insert version information of this software as string. """ function insert_version_info(db::SQLite.DB) - stmt = DBInterface.prepare( + stmt_insert_version = DBInterface.prepare( db, sql""" INSERT INTO versions VALUES (:converter) """, ) - DBInterface.execute(stmt, (; converter = pkgversion(@__MODULE__) |> string)) + DBInterface.execute( + stmt_insert_version, + (; converter = pkgversion(@__MODULE__) |> string), + ) nothing end @@ -28,12 +31,12 @@ function insert_qaqc_campaign_id(db::SQLite.DB) (DateTime(2024, 9, 10), DateTime(2024, 9, 12)), (DateTime(2024, 9, 30), DateTime(2024, 10, 4)), ] - stmt = DBInterface.prepare( + stmt_insert_campaigns = DBInterface.prepare( db, sql"INSERT INTO qaqc_campaigns VALUES (:id, :start_date, :end_date, :note)", ) DBInterface.executemany( - stmt, + stmt_insert_campaigns, ( id = campaigns, start_date = dates .|> (x -> x[1]) .|> string, @@ -58,7 +61,7 @@ function insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame) Symbol("立ち上がり [ns]") => ByRow(Float64) => Symbol("立ち上がり [ns]"), ) - stmt = DBInterface.prepare( + stmt_insert_positions = DBInterface.prepare( db, sql""" INSERT INTO qaqc_positions @@ -72,7 +75,7 @@ function insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame) """, ) DBInterface.executemany( - stmt, + stmt_insert_positions, ( id = 1:18, name = ["B-$i-$j" for i in 0:1 for j in 1:9], @@ -138,12 +141,12 @@ function add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame) end filter!(:motherboard_id => !=(999999), df) - stmt = DBInterface.prepare( + stmt_insert_psbid = DBInterface.prepare( db, sql"INSERT INTO ps_boards VALUES (:psbid, :daughterboardid)", ) DBInterface.executemany( - stmt, + stmt_insert_psbid, (psbid = df.motherboard_id, daughterboardid = df.daughterboard), ) @@ -426,7 +429,7 @@ function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame) dispatch_table = prepare_dispatch_table(dispatch_table) # TODO: provide datetime - stmt = DBInterface.prepare( + stmt_insert_dispatch = DBInterface.prepare( db, sql""" INSERT INTO qaqc_dispatch(qaqc_campaign_id, psb_id, source_place, destination, time) @@ -435,7 +438,7 @@ function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame) ) DBInterface.executemany( - stmt, + stmt_insert_dispatch, (campaign_id = dispatch_table.campaign_id, psboard_id = dispatch_table.psboard_id), ) @@ -717,7 +720,7 @@ See [`ClockParser.get_skew`](@ref) for parse detail. - `630_190`: broken file """ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) - stmt_insrt = DBInterface.prepare( + stmt_insert_skew_to_single_result = DBInterface.prepare( db, sql""" UPDATE qaqc_single_run_results @@ -726,7 +729,7 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) """, ) clk_files = - readdir("$logs_dir/main", join = true) |> + readdir(joinpath(logs_dir, "main"), join = true) |> filter(endswith("_clk.txt")) |> filter(!contains("nagoya")) @@ -743,7 +746,7 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) end DBInterface.execute( - stmt_insrt, + stmt_insert_skew_to_single_result, (skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]), ) end @@ -751,3 +754,82 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString) nothing end + +""" + add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) + +Extract QAQC results from slave log files for single runs. +Slave log files are expected to located in certain format under `logs_dir`. +""" +function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString) + exclude_runs = + ((runid = 51, reason = "clock only"), (runid = 175, reason = "broken files")) + + stmt_insert_slave_result_to_single_result = DBInterface.prepare( + db, + sql""" + UPDATE qaqc_single_run_results + SET + power_3v3d = :power_3v3d, + power_3v3a = :power_3v3a, + power_n3va = :power_n3va + WHERE + runid = :runid AND psboard_id = :psbid + """, + ) + runids = + DBInterface.execute( + db, + sql""" + SELECT id + FROM qaqc_runs + """, + ) |> Tables.columntable |> (tbl -> tbl.id) + + slave_files = + readdir(joinpath(logs_dir, "main"), join = true) |> + filter(contains(r"\d+_\d+\.txt")) + + DBInterface.transaction(db) do + for file in slave_files + psbid, runid, islongrun = + SlaveLogParser.get_psbid_runid_from_filename(basename(file)) + @assert !islongrun + + # exclusion + + exclude_cond = Iterators.filter(exclude_runs) do cond + runid == cond.runid + end + if !isempty(exclude_cond) + @debug "skipping runid = $(runid) for $(first(exclude_cond).reason)" + continue + end + + if !(runid in runids) + slave_result = SlaveLogParser.parse_slavelog_file(file) + @debug "runid: $(runid) not in run list (psbid: $(psbid))" + continue + end + + # main + + slave_result = SlaveLogParser.parse_slavelog_file(file) + + @assert length(slave_result.power) == 1 "Too many power results for single run" + + DBInterface.execute( + stmt_insert_slave_result_to_single_result, + (; + power_3v3d = slave_result.power[1].result_3v3d, + power_3v3a = slave_result.power[1].result_3v3a, + power_n3va = slave_result.power[1].result_n3va, + runid, + psbid, + ), + ) + end + end + + nothing +end diff --git a/src/parse_clock.jl b/src/parse_clock.jl index 92f71cc..dcdbb6d 100644 --- a/src/parse_clock.jl +++ b/src/parse_clock.jl @@ -19,7 +19,7 @@ Invalid cases are: - no measurement has >500 counts => "Clock skew out of range" """ function get_skew(file::T) where {T <: AbstractString} - @debug "file: $(file)" + # @debug "file: $(file)" lines = Iterators.Stateful(eachline(file)) was_0_before = false @@ -40,7 +40,7 @@ function get_skew(file::T) where {T <: AbstractString} return time end end - @debug "Clock skew out of range" + @debug "Clock skew out of range (file: $(file))" return missing end diff --git a/src/sql/create_table.sql b/src/sql/create_table.sql index 97fe4ab..cdb3e8a 100644 --- a/src/sql/create_table.sql +++ b/src/sql/create_table.sql @@ -22,6 +22,9 @@ CREATE TABLE qaqc_single_run_results ( reset INTEGER, qaqc_result INTEGER, lvds_tx_skew REAL, + power_3v3d REAL, + power_3v3a REAL, + power_n3va REAL, note TEXT, FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"), FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"), diff --git a/test/input/.gitignore b/test/input/.gitignore index 4641bcb..27eee19 100644 --- a/test/input/.gitignore +++ b/test/input/.gitignore @@ -9,3 +9,4 @@ slavelogs/main/* !slavelogs/main/525_245_longrun.txt !slavelogs/main/430_100.txt !slavelogs/main/364_88_longrun.txt +!slavelogs/main/127_172.txt diff --git a/test/runtests.jl b/test/runtests.jl index 3e855fd..049ce1c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -97,16 +97,25 @@ true || include("../src/PSBoardDataBase.jl") @test result.result_3v3d == 3.47 @test result.result_3v3a == 2.91 @test result.result_n3va == -3.01 + + lines = Iterators.Stateful( + Iterators.drop(eachline("./input/slavelogs/main/127_172.txt"), 17), + ) + result = PSBoardDataBase.SlaveLogParser.parse_power_section!(lines) + @test result.result_3v3d == 7.65 + @test result.result_3v3a == 3.80 + @test result.result_n3va == -2.87 + @test result.fpga_temp == -63.21 end @testset "Recov" begin lines = Iterators.Stateful( - Iterators.drop(eachline("./input/slavelogs/main/430_100.txt"), 1912) + Iterators.drop(eachline("./input/slavelogs/main/430_100.txt"), 1912), ) @test PSBoardDataBase.SlaveLogParser.parse_recov_section!(lines) lines = Iterators.Stateful( - Iterators.drop(eachline("./input/slavelogs/main/525_244.txt"), 1912) + Iterators.drop(eachline("./input/slavelogs/main/525_244.txt"), 1912), ) @test PSBoardDataBase.SlaveLogParser.parse_recov_section!(lines) @@ -240,6 +249,8 @@ true || include("../src/PSBoardDataBase.jl") @test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |> isnothing + @test PSBoardDataBase.add_slavelog_result(db, "input/slavelogs/") |> isnothing + run(`sqlitebrowser $dbpath`) @test PSBoardDataBase.create_database_from_exported_csvs(