mirror of
https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git
synced 2025-06-08 05:55:42 +09:00
new(slavelog parser): parser for asdtp section
- a lot of boilerplates... - there should be smarter way
This commit is contained in:
parent
9a4a4e8952
commit
3b490cd28c
8 changed files with 8938 additions and 0 deletions
|
|
@ -4,6 +4,7 @@ authors = ["Wataru Otsubo <wotsubo@icepp.s.u-tokyo.ac.jp>"]
|
|||
version = "0.2.0"
|
||||
|
||||
[deps]
|
||||
AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f"
|
||||
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
|
||||
DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
|
||||
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
|
||||
|
|
@ -12,6 +13,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
|
|||
Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
|
||||
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
|
||||
SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
|
||||
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
|
||||
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
|
||||
|
||||
[weakdeps]
|
||||
|
|
@ -21,12 +23,14 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
|
|||
PSBoardDataBaseInteractiveUtilsExt = "InteractiveUtils"
|
||||
|
||||
[compat]
|
||||
AutoHashEquals = "2.2"
|
||||
CSV = "0.10"
|
||||
DBInterface = "2"
|
||||
DataFrames = "1"
|
||||
Documenter = "1"
|
||||
Downloads = "1"
|
||||
SQLite = "1"
|
||||
StaticArrays = "1.9"
|
||||
Tables = "1"
|
||||
|
||||
[extras]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ using Dates
|
|||
|
||||
include("parse_qaqc_master_log.jl")
|
||||
include("parse_clock.jl")
|
||||
include("SlaveLogParser.jl")
|
||||
|
||||
include("create_table.jl")
|
||||
include("download_csv.jl")
|
||||
|
|
|
|||
361
src/SlaveLogParser.jl
Normal file
361
src/SlaveLogParser.jl
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
module SlaveLogParser
|
||||
|
||||
using StaticArrays
|
||||
using AutoHashEquals
|
||||
|
||||
const HEADER_QSPIP_START = "=============== Test QAPIp Start ==============="
|
||||
const HEADER_POWER_START = "=============== Test Power Start ==============="
|
||||
const HEADER_ASDTP_START = "=============== Test ASDTP Start ==============="
|
||||
const HEADER_RECOV_START = "=============== Test Recov Start ==============="
|
||||
HEADER_STARTS =
|
||||
[HEADER_QSPIP_START, HEADER_POWER_START, HEADER_ASDTP_START, HEADER_RECOV_START]
|
||||
|
||||
"""
|
||||
Indicate parser state.
|
||||
Default is `MODE_NONE`.
|
||||
In `MODE_NONE`, each line is feeded into parser to detect the start of each section.
|
||||
"""
|
||||
@enum SlaveLogSection begin
|
||||
MODE_NONE
|
||||
MODE_QSPIP
|
||||
MODE_POWER
|
||||
MODE_ASDTP
|
||||
MODE_RECOV
|
||||
end
|
||||
|
||||
struct PowerResult
|
||||
result_3v3d::Float64
|
||||
result_3v3a::Float64
|
||||
result_n3va::Float64
|
||||
fpga_temp::Float64
|
||||
|
||||
result::Bool
|
||||
end
|
||||
|
||||
struct SlaveLogResult end
|
||||
|
||||
"""
|
||||
get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64,Int64,Bool}
|
||||
|
||||
Extract info from slave log filename.
|
||||
"""
|
||||
function get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64, Int64, Bool}
|
||||
main, _ext = splitext(filename)
|
||||
parts = split(main, '_')
|
||||
psbid = parse(Int64, parts[1])
|
||||
runid = parse(Int64, parts[2])
|
||||
islongrun = if length(parts) == 3
|
||||
true
|
||||
else
|
||||
false
|
||||
end
|
||||
|
||||
(psbid, runid, islongrun)
|
||||
end
|
||||
|
||||
function is_valid_slavelog(filename::AbstractString)::Bool
|
||||
error("not yet implemented")
|
||||
end
|
||||
|
||||
"""
|
||||
detect_mode_start(line::AbstractString)
|
||||
|
||||
Detect [`SlaveLogSection`](@ref) from section starting header line.
|
||||
If the line doesn't match any section, returns `nothing`.
|
||||
"""
|
||||
function detect_mode_start(line::AbstractString)
|
||||
if line == HEADER_QSPIP_START
|
||||
MODE_QSPIP
|
||||
elseif line == HEADER_POWER_START
|
||||
MODE_POWER
|
||||
elseif line == HEADER_ASDTP_START
|
||||
MODE_ASDTP
|
||||
elseif line == HEADER_RECOV_START
|
||||
MODE_RECOV
|
||||
else
|
||||
nothing
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
detect_mode_start!(mode::SlaveLogSection, line::AbstractString)
|
||||
|
||||
Detect mode from the `line` and update `mode`.
|
||||
"""
|
||||
function detect_mode_start(mode::SlaveLogSection, line::AbstractString)
|
||||
newmode = detect_mode_start(line)
|
||||
if !isnothing(newmode)
|
||||
mode = newmode
|
||||
end
|
||||
mode
|
||||
end
|
||||
|
||||
"""
|
||||
parse_qspip_section(lines::Base.Iterators.Stateful)
|
||||
|
||||
Parse QSPIp section of given stateful iterator of log.
|
||||
|
||||
# Args
|
||||
- `lines`: Stateful iterator of slave log file lines
|
||||
"""
|
||||
function parse_qspip_section!(lines::Base.Iterators.Stateful)
|
||||
# TODO
|
||||
nothing
|
||||
end
|
||||
|
||||
"""
|
||||
parse_power_section(lines::Base.Iterators.Stateful)
|
||||
|
||||
Parse Power section of given stateful iterator of log.
|
||||
|
||||
# Args
|
||||
- `lines`: Stateful iterator of slave log file lines
|
||||
"""
|
||||
function parse_power_section!(lines::Base.Iterators.Stateful)
|
||||
# TODO
|
||||
nothing
|
||||
end
|
||||
|
||||
"""
|
||||
Measurement result for asic in asdtp test.
|
||||
"""
|
||||
@auto_hash_equals struct AsdtpMeasurement
|
||||
before::Float64
|
||||
current::Float64
|
||||
next::Float64
|
||||
end
|
||||
|
||||
AsdtpMeasurement(x::NTuple{3, <:Real}) = AsdtpMeasurement(x...)
|
||||
|
||||
function Base.parse(::Type{AsdtpMeasurement}, s::AbstractString)
|
||||
v = split(s, ':')
|
||||
@assert length(v) == 3
|
||||
AsdtpMeasurement(parse(Float64, v[1]), parse(Float64, v[2]), parse(Float64, v[3]))
|
||||
end
|
||||
|
||||
@auto_hash_equals struct AsdtpResult
|
||||
reconfig_done::Int64
|
||||
always_hit_flag::Int64
|
||||
autoreconfig::Union{Bool, Missing}
|
||||
asdtp_main::Union{Vector{Vector{AsdtpMeasurement}}, Missing}
|
||||
asdtp_reset::Int64
|
||||
asdtp_total::Int64
|
||||
reconfig_done_2::Int64
|
||||
always_hit_flag_2::Int64
|
||||
si_done::UInt64
|
||||
lolb_in::UInt64
|
||||
ppconfig_done::UInt64
|
||||
ppconfig_error::UInt64
|
||||
pllld_fail_counter::UInt64
|
||||
ppconfig_fail_counter::UInt64
|
||||
pp_pllds::MVector{8, UInt32}
|
||||
end
|
||||
|
||||
"""
|
||||
parse_asdtp_section(lines::Base.Iterators.Stateful)
|
||||
|
||||
Parse ASDTP section of given stateful iterator of log.
|
||||
|
||||
# Args
|
||||
- `lines`: Stateful iterator of slave log file lines
|
||||
"""
|
||||
function parse_asdtp_section!(lines::Base.Iterators.Stateful)
|
||||
line = popfirst!(lines)
|
||||
result_reconfig_done = let
|
||||
m = match(r"^reconfig_done = (\d+)$", line)
|
||||
parse(Int64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_always_hit_flag = let
|
||||
m = match(r"^always_hit_flag = (\d+)$", line)
|
||||
parse(Int64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_autoreconfig = if line == "Autoreconfig done"
|
||||
true
|
||||
elseif line == "Autoreconfig fail"
|
||||
false
|
||||
else
|
||||
missing
|
||||
end
|
||||
|
||||
result_asdtp_main = if !ismissing(result_autoreconfig) && result_autoreconfig
|
||||
line_count = 0
|
||||
line_count += 1
|
||||
results = map(_ -> AsdtpMeasurement[], 1:8)
|
||||
for asic_id in 1:8
|
||||
header_line = "----PP$(asic_id)----"
|
||||
while line != header_line
|
||||
line = popfirst!(lines)
|
||||
line_count += 1
|
||||
end
|
||||
for _ in 1:32
|
||||
line = popfirst!(lines)
|
||||
line_count += 1
|
||||
mes = parse(AsdtpMeasurement, line)
|
||||
push!(results[asic_id], mes)
|
||||
end
|
||||
end
|
||||
@assert length(results[1]) == 32 "unexpected length: $(length(results[1]))"
|
||||
|
||||
line = popfirst!(lines)
|
||||
@assert line == "100"
|
||||
results
|
||||
else
|
||||
missing
|
||||
end
|
||||
|
||||
line = popfirst!(lines)
|
||||
result_asdtp_reset, result_asdtp_total = let
|
||||
m = match(r"^ASDTP : (\d+) times reset : result = (\d+)$", line)
|
||||
parse(Int64, m[1]), parse(Int64, m[2])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_reconfig_done_2 = let
|
||||
m = match(r"^reconfig_done = (\d+)$", line)
|
||||
parse(Int64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_always_hit_flag_2 = let
|
||||
m = match(r"^always_hit_flag = (\d+)$", line)
|
||||
parse(Int64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
@assert line == ""
|
||||
|
||||
line = popfirst!(lines)
|
||||
@assert line == "------- Done check -------" "actual line: $line"
|
||||
line = popfirst!(lines)
|
||||
result_si_done = let
|
||||
m = match(r"^Si_done = (0x[[:xdigit:]]+)$", line)
|
||||
parse(UInt64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_lolb_in = let
|
||||
m = match(r"^LOLB_in = (0x[[:xdigit:]]+)$", line)
|
||||
parse(UInt64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_ppconfig_done = let
|
||||
m = match(r"^PPconfig_done = (0x[[:xdigit:]]+)$", line)
|
||||
parse(UInt64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_ppconfig_error = let
|
||||
m = match(r"^PPconfig_error = (0x[[:xdigit:]]+)$", line)
|
||||
parse(UInt64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_pllld_fail_counter = let
|
||||
m = match(r"^PLLLD_fail_counter = (0x[[:xdigit:]]+)$", line)
|
||||
parse(UInt64, m[1])
|
||||
end
|
||||
line = popfirst!(lines)
|
||||
result_ppconfig_fail_counter = let
|
||||
m = match(r"^PPconfig_fail_counter = (0x[[:xdigit:]]+)$", line)
|
||||
parse(UInt64, m[1])
|
||||
end
|
||||
result_pp_pllds = MVector{8, UInt32}(undef)
|
||||
for ppid in 1:8
|
||||
line = popfirst!(lines)
|
||||
m = match(Regex("^PP$(ppid)_PLLLD = (0x[[:xdigit:]]+)\$"), line)
|
||||
result_pp_pllds[ppid] = parse(UInt32, m[1])
|
||||
end
|
||||
|
||||
AsdtpResult(
|
||||
result_reconfig_done,
|
||||
result_always_hit_flag,
|
||||
result_autoreconfig,
|
||||
result_asdtp_main,
|
||||
result_asdtp_reset,
|
||||
result_asdtp_total,
|
||||
result_reconfig_done_2,
|
||||
result_always_hit_flag_2,
|
||||
result_si_done,
|
||||
result_lolb_in,
|
||||
result_ppconfig_done,
|
||||
result_ppconfig_error,
|
||||
result_pllld_fail_counter,
|
||||
result_ppconfig_fail_counter,
|
||||
result_pp_pllds,
|
||||
)
|
||||
end
|
||||
|
||||
"""
|
||||
parse_qspip_section(lines::Base.Iterators.Stateful)
|
||||
|
||||
Parse Recov section of given stateful iterator of log.
|
||||
|
||||
# Args
|
||||
- `lines`: Stateful iterator of slave log file lines
|
||||
|
||||
# Return
|
||||
- `nothing`: if failed to parse
|
||||
- `true`: if successed
|
||||
- `false`
|
||||
"""
|
||||
function parse_recov_section!(lines::Base.Iterators.Stateful)
|
||||
line = popfirst!(lines)
|
||||
if startswith("====")(line)
|
||||
line = popfirst!(lines)
|
||||
end
|
||||
m = match(r"Test Recov Result = (\d+)", line)
|
||||
if isnothing(m)
|
||||
return nothing
|
||||
else
|
||||
return m[1] == "1"
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
parse_slavelog_file(filename::AbstractString)
|
||||
|
||||
Main function to parse slave log file.
|
||||
"""
|
||||
function parse_slavelog_file(filename::AbstractString)
|
||||
lines_iter = Iterators.Stateful(eachline(filename))
|
||||
|
||||
asdtp_results = Any[]
|
||||
|
||||
mode::SlaveLogSection = MODE_NONE
|
||||
# main loop
|
||||
while !isempty(lines_iter)
|
||||
# each sections
|
||||
if mode == MODE_NONE
|
||||
line = popfirst!(lines_iter)
|
||||
mode = detect_mode_start(mode, line)
|
||||
elseif mode == MODE_QSPIP
|
||||
parse_qspip_section!(lines_iter)
|
||||
mode = MODE_NONE
|
||||
elseif mode == MODE_POWER
|
||||
parse_power_section!(lines_iter)
|
||||
mode = MODE_NONE
|
||||
elseif mode == MODE_ASDTP
|
||||
result = parse_asdtp_section!(lines_iter)
|
||||
push!(asdtp_results, result)
|
||||
mode = MODE_NONE
|
||||
elseif mode == MODE_RECOV
|
||||
parse_recov_section!(lines_iter)
|
||||
mode = MODE_NONE
|
||||
end
|
||||
end
|
||||
|
||||
@info "Finished"
|
||||
return asdtp_results
|
||||
end
|
||||
|
||||
function eff99_count_map(asdtp_results)
|
||||
# try(100) × channel(8) × channel(32)
|
||||
@assert length(asdtp_results) == 100
|
||||
@assert length(asdtp_results[begin]) == 8
|
||||
@assert length(asdtp_results[begin][begin]) == 32
|
||||
map(1:8) do i_asic
|
||||
map(1:32) do i_channel
|
||||
sum(1:100) do i_try
|
||||
asdtp_results[i_try][i_asic][i_channel] != AsdtpMeasurement(0, 1, 0)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
3
test/input/.gitignore
vendored
3
test/input/.gitignore
vendored
|
|
@ -5,3 +5,6 @@ slavelogs/main/*
|
|||
!slavelogs/main/448_103_clk.txt
|
||||
!slavelogs/main/444_103_clk.txt
|
||||
!slavelogs/main/209_51_clk.txt
|
||||
!slavelogs/main/525_244.txt
|
||||
!slavelogs/main/525_245_longrun.txt
|
||||
!slavelogs/main/430_100.txt
|
||||
|
|
|
|||
2567
test/input/slavelogs/main/430_100.txt
Normal file
2567
test/input/slavelogs/main/430_100.txt
Normal file
File diff suppressed because it is too large
Load diff
2302
test/input/slavelogs/main/525_244.txt
Normal file
2302
test/input/slavelogs/main/525_244.txt
Normal file
File diff suppressed because it is too large
Load diff
3641
test/input/slavelogs/main/525_245_longrun.txt
Normal file
3641
test/input/slavelogs/main/525_245_longrun.txt
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,5 +1,6 @@
|
|||
using Test
|
||||
using PSBoardDataBase
|
||||
using StaticArrays
|
||||
using CSV, DataFrames
|
||||
using SQLite, DBInterface
|
||||
using Dates
|
||||
|
|
@ -45,6 +46,64 @@ true || include("../src/PSBoardDataBase.jl")
|
|||
) == 22
|
||||
end
|
||||
|
||||
@testset "Slave Log parser" begin
|
||||
lines = Iterators.Stateful(
|
||||
Iterators.drop(eachline("./input/slavelogs/main/430_100.txt"), 2280),
|
||||
)
|
||||
result_asdtp = PSBoardDataBase.SlaveLogParser.parse_asdtp_section!(lines)
|
||||
# target = PSBoardDataBase.SlaveLogParser.AsdtpResult(
|
||||
# 1,
|
||||
# 0,
|
||||
# true,
|
||||
# fill(PSBoardDataBase.SlaveLogParser.AsdtpMeasurement.(fill((0, 1, 0), 32)), 8),
|
||||
# 0,
|
||||
# 1,
|
||||
# 1,
|
||||
# 0,
|
||||
# 0x1,
|
||||
# 0x1,
|
||||
# 0xff,
|
||||
# 0x0,
|
||||
# 0x0,
|
||||
# 0x0,
|
||||
# MVector((0x1001, 0x2001, 0x3001, 0x4001, 0x5001, 0x6001, 0x7001, 0x8001)),
|
||||
# )
|
||||
# @info "" target == result_asdtp zip(result_asdtp.pp_pllds, target.pp_pllds) |>
|
||||
# collect result_asdtp.pp_pllds == target.pp_pllds Iterators.filter(
|
||||
# x -> x[1] != x[2],
|
||||
# zip(result_asdtp.pp_pllds, target.pp_pllds),
|
||||
# ) |> collect
|
||||
# @info "" map(fieldnames(typeof(result_asdtp))) do field
|
||||
# (; field, diff = getfield(result_asdtp, field) == getfield(target, field))
|
||||
# end
|
||||
|
||||
@test result_asdtp == PSBoardDataBase.SlaveLogParser.AsdtpResult(
|
||||
1,
|
||||
0,
|
||||
true,
|
||||
fill(PSBoardDataBase.SlaveLogParser.AsdtpMeasurement.(fill((0, 1, 0), 32)), 8),
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0x1,
|
||||
0x1,
|
||||
0xff,
|
||||
0x0,
|
||||
0x0,
|
||||
0x0,
|
||||
MVector((0x1001, 0x2001, 0x3001, 0x4001, 0x5001, 0x6001, 0x7001, 0x8001)),
|
||||
)
|
||||
@info "pass"
|
||||
lines = Iterators.Stateful(
|
||||
Iterators.drop(eachline("./input/slavelogs/main/525_244.txt"), 2280),
|
||||
)
|
||||
PSBoardDataBase.SlaveLogParser.parse_asdtp_section!(lines)
|
||||
# PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
|
||||
# "./input/slavelogs/main/525_244.txt",
|
||||
# )
|
||||
end
|
||||
|
||||
@testset "Download data csv" begin
|
||||
out = tempname()
|
||||
@test CSV.read(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue