[new] Slave clock parser and function to insert into database

This commit is contained in:
Wataru Otsubo 2024-10-01 11:24:44 +02:00
parent 1422798c3a
commit bf19ee3167
10 changed files with 1861 additions and 3 deletions

View file

@ -10,13 +10,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Add `versions` table to store version information of converter(this software)
- Add `skew` columnt to `qaqc_positions` table
- Add `skew` column to `qaqc_positions` table
- Add `DownloadCSVs` module and functions which downloads the latest CSVs from Google sheets
- Add `ClockParser` module and `get_skew` function
- Add `lvds_tx_skew` column to `qaqc_single_run_result` table and related functions in `import_data.jl`
### Changed
- Set download functions in `DownloadCSVs` as default CSV locations in `create_database_from_exported_csvs`
- Replaced CSV files used in test to newly add `DownloadCSVs` functions
- `create_database_from_exported_csvs` now requires `slavelog_dir` to get skew from slave logs
### Deleted

View file

@ -1,6 +1,14 @@
```@meta
CurrentModule = PSBoardDataBase
```
# Index
```@contents
Pages = ["about_software.md"]
Depth = 4
```
# このリポジトリのソフトウェアについての説明
このリポジトリにあるのは、JATHub masterのログファイル、及びGoogle SheetsからエクスポートしたCSVファイルからデータベースを作成するためのコードである。
@ -81,3 +89,9 @@ Modules = [QaqcMasterLog]
```@autodocs
Modules = [DownloadCSVs]
```
## `ClockParser`
```@autodocs
Modules = [ClockParser]
```

View file

@ -8,6 +8,7 @@ using DataFrames
using Dates
include("parse_qaqc_master_log.jl")
include("parse_clock.jl")
include("create_table.jl")
include("download_csv.jl")
@ -22,6 +23,7 @@ include("import_data.jl")
hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
masterlog_dir::AbstractString,
slavelog_dir::AbstractString,
)
Create database at `dbpath` and import data from CSV and master log files.
@ -33,7 +35,8 @@ Create database at `dbpath` and import data from CSV and master log files.
- `dispatch_csv`: CSV of dispatch lists exported from the Google sheets database
- `hundred_csv`: CSV of 100 tests results exported from the Google sheets database
- `jathubs_csv`: CSV for jathub list used in QAQC. Used to add skew.
- `masterlog_dir`: path to the directory (`log`) where all JATHub master log is stored
- `masterlog_dir`: path to the directory (`log`) where all JATHub master logs are stored
- `slavelog_dir`: path to the directory where all JATHub slave logs are stored
"""
function create_database_from_exported_csvs(
dbpath::AbstractString;
@ -43,6 +46,7 @@ function create_database_from_exported_csvs(
hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
masterlog_dir::AbstractString,
slavelog_dir::AbstractString,
)
db = create_database(dbpath)
single_result_df = CSV.read(single_run_csv, DataFrame)
@ -59,6 +63,7 @@ function create_database_from_exported_csvs(
add_qaqc_dispatch(db, dispatch_table)
add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
add_qaqc_100test_result(db, extra_100test_result_df)
add_skew_from_slave_clk_logs(db, slavelog_dir)
db
end

View file

@ -702,3 +702,43 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
nothing
end
"""
add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
Insert skew measurement result from slave logs with name `psbid_runid_clk.txt`.
See [`ClockParser.get_skew`](@ref) for parse detail.
# Abnormal logs:
- `48_nagoya_irradition_...`: skipped
"""
function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
stmt_insrt = DBInterface.prepare(
db,
sql"""
UPDATE qaqc_single_run_results
SET lvds_tx_skew = :skew
WHERE runid = :runid AND psboard_id = :psbid
""",
)
clk_files =
readdir("$logs_dir/main", join = true) |>
filter(endswith("_clk.txt")) |>
filter(!contains("nagoya"))
DBInterface.transaction(db) do
for file in clk_files
m = match(r"^(?<psbid>\d+)_(?<runid>\d+)_clk.txt$", splitdir(file) |> last)
if isnothing(m)
error("Invalid filename $(file)")
end
DBInterface.execute(
stmt_insrt,
(skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]),
)
end
end
nothing
end

46
src/parse_clock.jl Normal file
View file

@ -0,0 +1,46 @@
module ClockParser
export get_skew
function _parse_line(line::AbstractString)
time, high, _ = split(line)
parse(Float64, time), parse(Float64, high)
end
"""
get_skew(file::T) where {T <: AbstractString} -> Union{Float64, Missing}
Get skew from clock result file `file`.
It returns `missing` for invalid files.
To see the detailed reason, increase log level to `DEBUG`.
Invalid cases are:
- first line has >0 counts => "Unexpected first line"
- no measurement has >500 counts => "Clock skew out of range"
"""
function get_skew(file::T) where {T <: AbstractString}
lines = Iterators.Stateful(eachline(file))
first_line = popfirst!(lines)
# criteria: https://gitlab.cern.ch/dhashimo/PS_Board_QAQC/-/blob/master/Software_Test/project_auto_reset/QAQC_test_func.cxx?ref_type=heads#L208
let
time, high = _parse_line(first_line)
if high > 0
@debug "Unexpected first line" file
return missing
elseif high > 500
return time
end
end
time_and_counts = Iterators.map(_parse_line, lines)
for (time, high) in time_and_counts
if high > 500
return time
end
end
@debug "Clock skew out of range"
return missing
end
end # module ClockParser

View file

@ -21,6 +21,7 @@ CREATE TABLE qaqc_single_run_results (
asdtp INTEGER,
reset INTEGER,
qaqc_result INTEGER,
lvds_tx_skew REAL,
note TEXT,
FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
@ -114,8 +115,8 @@ AS
qaqc_single_run_results.asdtp,
qaqc_single_run_results.reset,
qaqc_single_run_results.qaqc_result,
qaqc_single_run_results.lvds_tx_skew - qaqc_positions.rising_ns AS lvds_tx_skew,
qaqc_runs.shiftscript_ver,
qaqc_runs.shifter,
qaqc_single_run_results.note AS result_note
FROM
ps_boards,

View file

@ -1,2 +1,5 @@
log/*
slavelogs/main/*
!log/57_long.log
!slavelogs/main/230_51_clk.txt
!slavelogs/main/448_103_clk.txt

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,337 @@
10.000000000000000 0.000000 0.000000
10.017857142857142 0.000000 0.000000
10.035714285714286 0.000000 0.000000
10.053571428571429 0.000000 0.000000
10.071428571428571 0.000000 0.000000
10.089285714285714 0.000000 0.000000
10.107142857142858 0.000000 0.000000
10.125000000000000 0.000000 0.000000
10.142857142857142 0.000000 0.000000
10.160714285714286 0.000000 0.000000
10.178571428571429 0.000000 0.000000
10.196428571428571 0.000000 0.000000
10.214285714285714 0.000000 0.000000
10.232142857142858 0.000000 0.000000
10.250000000000000 0.000000 0.000000
10.267857142857142 0.000000 0.000000
10.285714285714286 0.000000 0.000000
10.303571428571429 0.000000 0.000000
10.321428571428571 0.000000 0.000000
10.339285714285714 0.000000 0.000000
10.357142857142858 0.000000 0.000000
10.375000000000000 0.000000 0.000000
10.392857142857142 0.000000 0.000000
10.410714285714286 0.000000 0.000000
10.428571428571429 0.000000 0.000000
10.446428571428571 0.000000 0.000000
10.464285714285714 0.000000 0.000000
10.482142857142858 0.000000 0.000000
10.500000000000000 0.000000 0.000000
10.517857142857142 0.000000 0.000000
10.535714285714286 0.000000 0.000000
10.553571428571429 0.000000 0.000000
10.571428571428571 0.000000 0.000000
10.589285714285714 0.000000 0.000000
10.607142857142858 0.000000 0.000000
10.625000000000000 0.000000 0.000000
10.642857142857142 0.000000 0.000000
10.660714285714286 0.000000 0.000000
10.678571428571429 0.000000 0.000000
10.696428571428571 0.000000 0.000000
10.714285714285714 0.000000 0.000000
10.732142857142858 0.000000 0.000000
10.750000000000000 0.000000 0.000000
10.767857142857142 0.000000 0.000000
10.785714285714286 0.000000 0.000000
10.803571428571429 0.000000 0.000000
10.821428571428571 0.000000 0.000000
10.839285714285714 0.000000 0.000000
10.857142857142858 0.000000 0.000000
10.875000000000000 0.000000 0.000000
10.892857142857142 0.000000 0.000000
10.910714285714286 0.000000 0.000000
10.928571428571429 0.000000 0.000000
10.946428571428571 0.000000 0.000000
10.964285714285714 0.000000 0.000000
10.982142857142858 0.000000 0.000000
11.000000000000000 0.000000 0.000000
11.017857142857142 0.000000 0.000000
11.035714285714286 0.000000 0.000000
11.053571428571429 0.000000 0.000000
11.071428571428571 0.000000 0.000000
11.089285714285714 0.000000 0.000000
11.107142857142858 0.000000 0.000000
11.125000000000000 0.000000 0.000000
11.142857142857142 0.000000 0.000000
11.160714285714286 0.000000 0.000000
11.178571428571429 0.000000 0.000000
11.196428571428571 0.000000 0.000000
11.214285714285714 0.000000 0.000000
11.232142857142858 0.000000 0.000000
11.250000000000000 0.000000 0.000000
11.267857142857142 0.000000 0.000000
11.285714285714286 0.000000 0.000000
11.303571428571429 0.000000 0.000000
11.321428571428571 0.000000 0.000000
11.339285714285714 0.000000 0.000000
11.357142857142858 0.000000 0.000000
11.375000000000000 0.000000 0.000000
11.392857142857142 0.000000 0.000000
11.410714285714286 0.000000 0.000000
11.428571428571429 0.000000 0.000000
11.446428571428571 0.000000 0.000000
11.464285714285714 0.000000 0.000000
11.482142857142858 0.000000 0.000000
11.500000000000000 0.000000 0.000000
11.517857142857142 0.000000 0.000000
11.535714285714286 0.000000 0.000000
11.553571428571429 0.000000 0.000000
11.571428571428571 0.000000 0.000000
11.589285714285714 0.000000 0.000000
11.607142857142858 0.000000 0.000000
11.625000000000000 0.000000 0.000000
11.642857142857142 0.000000 0.000000
11.660714285714286 0.000000 0.000000
11.678571428571429 0.000000 0.000000
11.696428571428571 0.000000 0.000000
11.714285714285714 0.000000 0.000000
11.732142857142858 0.000000 0.000000
11.750000000000000 0.000000 0.000000
11.767857142857142 0.000000 0.000000
11.785714285714286 0.000000 0.000000
11.803571428571429 0.000000 0.000000
11.821428571428571 0.000000 0.000000
11.839285714285714 0.000000 0.000000
11.857142857142858 0.000000 0.000000
11.875000000000000 0.000000 0.000000
11.892857142857142 0.000000 0.000000
11.910714285714286 0.000000 0.000000
11.928571428571429 0.000000 0.000000
11.946428571428571 0.000000 0.000000
11.964285714285714 0.000000 0.000000
11.982142857142858 0.000000 0.000000
12.000000000000000 501.000000 0.000000
12.017857142857142 865.000000 0.000000
12.035714285714286 995.000000 0.000000
12.053571428571429 1000.000000 0.000000
12.071428571428571 1000.000000 0.000000
12.089285714285714 1000.000000 0.000000
12.107142857142858 1000.000000 0.000000
12.125000000000000 1000.000000 0.000000
12.142857142857142 1000.000000 0.000000
12.160714285714286 1000.000000 0.000000
12.178571428571429 1000.000000 0.000000
12.196428571428571 1000.000000 0.000000
12.214285714285714 1000.000000 0.000000
12.232142857142858 1000.000000 0.000000
12.250000000000000 1000.000000 0.000000
12.267857142857142 1000.000000 0.000000
12.285714285714286 1000.000000 0.000000
12.303571428571429 1000.000000 0.000000
12.321428571428571 1000.000000 0.000000
12.339285714285714 1000.000000 0.000000
12.357142857142858 1000.000000 0.000000
12.375000000000000 1000.000000 0.000000
12.392857142857142 1000.000000 0.000000
12.410714285714286 1000.000000 0.000000
12.428571428571429 1000.000000 0.000000
12.446428571428571 1000.000000 0.000000
12.464285714285714 1000.000000 0.000000
12.482142857142858 1000.000000 0.000000
12.500000000000000 1000.000000 0.000000
12.517857142857142 1000.000000 0.000000
12.535714285714286 1000.000000 0.000000
12.553571428571429 1000.000000 0.000000
12.571428571428571 1000.000000 0.000000
12.589285714285714 1000.000000 0.000000
12.607142857142858 1000.000000 0.000000
12.625000000000000 1000.000000 0.000000
12.642857142857142 1000.000000 0.000000
12.660714285714286 1000.000000 0.000000
12.678571428571429 1000.000000 0.000000
12.696428571428571 1000.000000 0.000000
12.714285714285714 1000.000000 0.000000
12.732142857142858 1000.000000 0.000000
12.750000000000000 1000.000000 0.000000
12.767857142857142 1000.000000 0.000000
12.785714285714286 1000.000000 0.000000
12.803571428571429 1000.000000 0.000000
12.821428571428571 1000.000000 0.000000
12.839285714285714 1000.000000 0.000000
12.857142857142858 1000.000000 0.000000
12.875000000000000 1000.000000 0.000000
12.892857142857142 1000.000000 0.000000
12.910714285714286 1000.000000 0.000000
12.928571428571429 1000.000000 0.000000
12.946428571428571 1000.000000 0.000000
12.964285714285714 1000.000000 0.000000
12.982142857142858 1000.000000 0.000000
13.000000000000000 1000.000000 0.000000
13.017857142857142 1000.000000 0.000000
13.035714285714286 1000.000000 0.000000
13.053571428571429 1000.000000 0.000000
13.071428571428571 1000.000000 0.000000
13.089285714285714 1000.000000 0.000000
13.107142857142858 1000.000000 0.000000
13.125000000000000 1000.000000 0.000000
13.142857142857142 1000.000000 0.000000
13.160714285714286 1000.000000 0.000000
13.178571428571429 1000.000000 0.000000
13.196428571428571 1000.000000 0.000000
13.214285714285714 1000.000000 0.000000
13.232142857142858 1000.000000 0.000000
13.250000000000000 1000.000000 0.000000
13.267857142857142 1000.000000 0.000000
13.285714285714286 1000.000000 0.000000
13.303571428571429 1000.000000 0.000000
13.321428571428571 1000.000000 0.000000
13.339285714285714 1000.000000 0.000000
13.357142857142858 1000.000000 0.000000
13.375000000000000 1000.000000 0.000000
13.392857142857142 1000.000000 0.000000
13.410714285714286 1000.000000 0.000000
13.428571428571429 1000.000000 0.000000
13.446428571428571 1000.000000 0.000000
13.464285714285714 1000.000000 0.000000
13.482142857142858 1000.000000 0.000000
13.500000000000000 1000.000000 0.000000
13.517857142857142 1000.000000 0.000000
13.535714285714286 1000.000000 0.000000
13.553571428571429 1000.000000 0.000000
13.571428571428571 1000.000000 0.000000
13.589285714285714 1000.000000 0.000000
13.607142857142858 1000.000000 0.000000
13.625000000000000 1000.000000 0.000000
13.642857142857142 1000.000000 0.000000
13.660714285714286 1000.000000 0.000000
13.678571428571429 1000.000000 0.000000
13.696428571428571 1000.000000 0.000000
13.714285714285714 1000.000000 0.000000
13.732142857142858 1000.000000 0.000000
13.750000000000000 1000.000000 0.000000
13.767857142857142 1000.000000 0.000000
13.785714285714286 1000.000000 0.000000
13.803571428571429 1000.000000 0.000000
13.821428571428571 1000.000000 0.000000
13.839285714285714 1000.000000 0.000000
13.857142857142858 1000.000000 0.000000
13.875000000000000 1000.000000 0.000000
13.892857142857142 1000.000000 0.000000
13.910714285714286 1000.000000 0.000000
13.928571428571429 1000.000000 0.000000
13.946428571428571 1000.000000 0.000000
13.964285714285714 1000.000000 0.000000
13.982142857142858 1000.000000 0.000000
14.000000000000000 1000.000000 0.000000
14.017857142857142 1000.000000 0.000000
14.035714285714286 1000.000000 0.000000
14.053571428571429 1000.000000 0.000000
14.071428571428571 1000.000000 0.000000
14.089285714285714 1000.000000 0.000000
14.107142857142858 1000.000000 0.000000
14.125000000000000 1000.000000 0.000000
14.142857142857142 1000.000000 0.000000
14.160714285714286 1000.000000 0.000000
14.178571428571429 1000.000000 0.000000
14.196428571428571 1000.000000 0.000000
14.214285714285714 1000.000000 0.000000
14.232142857142858 1000.000000 0.000000
14.250000000000000 1000.000000 0.000000
14.267857142857142 1000.000000 0.000000
14.285714285714286 1000.000000 0.000000
14.303571428571429 1000.000000 0.000000
14.321428571428571 1000.000000 0.000000
14.339285714285714 1000.000000 0.000000
14.357142857142858 1000.000000 0.000000
14.375000000000000 1000.000000 0.000000
14.392857142857142 1000.000000 0.000000
14.410714285714286 1000.000000 0.000000
14.428571428571429 1000.000000 0.000000
14.446428571428571 1000.000000 0.000000
14.464285714285714 1000.000000 0.000000
14.482142857142858 1000.000000 0.000000
14.500000000000000 1000.000000 0.000000
14.517857142857142 1000.000000 0.000000
14.535714285714286 1000.000000 0.000000
14.553571428571429 1000.000000 0.000000
14.571428571428571 1000.000000 0.000000
14.589285714285714 1000.000000 0.000000
14.607142857142858 1000.000000 0.000000
14.625000000000000 1000.000000 0.000000
14.642857142857142 1000.000000 0.000000
14.660714285714286 1000.000000 0.000000
14.678571428571429 1000.000000 0.000000
14.696428571428571 1000.000000 0.000000
14.714285714285714 1000.000000 0.000000
14.732142857142858 1000.000000 0.000000
14.750000000000000 1000.000000 0.000000
14.767857142857142 1000.000000 0.000000
14.785714285714286 1000.000000 0.000000
14.803571428571429 1000.000000 0.000000
14.821428571428571 1000.000000 0.000000
14.839285714285714 1000.000000 0.000000
14.857142857142858 1000.000000 0.000000
14.875000000000000 1000.000000 0.000000
14.892857142857142 1000.000000 0.000000
14.910714285714286 1000.000000 0.000000
14.928571428571429 1000.000000 0.000000
14.946428571428571 1000.000000 0.000000
14.964285714285714 1000.000000 0.000000
14.982142857142858 1000.000000 0.000000
15.000000000000000 1000.000000 0.000000
15.017857142857142 1000.000000 0.000000
15.035714285714286 1000.000000 0.000000
15.053571428571429 1000.000000 0.000000
15.071428571428571 1000.000000 0.000000
15.089285714285714 1000.000000 0.000000
15.107142857142858 1000.000000 0.000000
15.125000000000000 1000.000000 0.000000
15.142857142857142 1000.000000 0.000000
15.160714285714286 1000.000000 0.000000
15.178571428571429 1000.000000 0.000000
15.196428571428571 1000.000000 0.000000
15.214285714285714 1000.000000 0.000000
15.232142857142858 1000.000000 0.000000
15.250000000000000 1000.000000 0.000000
15.267857142857142 1000.000000 0.000000
15.285714285714286 1000.000000 0.000000
15.303571428571429 1000.000000 0.000000
15.321428571428571 1000.000000 0.000000
15.339285714285714 1000.000000 0.000000
15.357142857142858 1000.000000 0.000000
15.375000000000000 1000.000000 0.000000
15.392857142857142 1000.000000 0.000000
15.410714285714286 1000.000000 0.000000
15.428571428571429 1000.000000 0.000000
15.446428571428571 1000.000000 0.000000
15.464285714285714 1000.000000 0.000000
15.482142857142858 1000.000000 0.000000
15.500000000000000 1000.000000 0.000000
15.517857142857142 1000.000000 0.000000
15.535714285714286 1000.000000 0.000000
15.553571428571429 1000.000000 0.000000
15.571428571428571 1000.000000 0.000000
15.589285714285714 1000.000000 0.000000
15.607142857142858 1000.000000 0.000000
15.625000000000000 1000.000000 0.000000
15.642857142857142 1000.000000 0.000000
15.660714285714286 1000.000000 0.000000
15.678571428571429 1000.000000 0.000000
15.696428571428571 1000.000000 0.000000
15.714285714285714 1000.000000 0.000000
15.732142857142858 1000.000000 0.000000
15.750000000000000 1000.000000 0.000000
15.767857142857142 1000.000000 0.000000
15.785714285714286 1000.000000 0.000000
15.803571428571429 1000.000000 0.000000
15.821428571428571 1000.000000 0.000000
15.839285714285714 1000.000000 0.000000
15.857142857142858 1000.000000 0.000000
15.875000000000000 1000.000000 0.000000
15.892857142857142 1000.000000 0.000000
15.910714285714286 1000.000000 0.000000
15.928571428571429 1000.000000 0.000000
15.946428571428571 1000.000000 0.000000
15.964285714285714 1000.000000 0.000000
15.982142857142858 1000.000000 0.000000
16.000000000000000 1000.000000 0.000000

View file

@ -16,6 +16,11 @@ true || include("../src/PSBoardDataBase.jl")
@test masterlog_metadata.shiftscript_version == v"1.0.2"
end
@testset "parse clk log" begin
@test PSBoardDataBase.ClockParser.get_skew("input/slavelogs/main/230_51_clk.txt") |> ismissing
@test PSBoardDataBase.ClockParser.get_skew("input/slavelogs/main/448_103_clk.txt") 12.000000000000000
end
@testset "Download data csv" begin
out = tempname()
@test CSV.read(
@ -100,6 +105,9 @@ true || include("../src/PSBoardDataBase.jl")
@test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df) |>
isnothing
@test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |>
isnothing
run(`sqlitebrowser $dbpath`)
end
end