new(slavelog parser): parser for asdtp section

- a lot of boilerplates...
- there should be smarter way

Project.toml

@@ -4,6 +4,7 @@ authors = ["Wataru Otsubo <wotsubo@icepp.s.u-tokyo.ac.jp>"]
 version = "0.2.0"
 
 [deps]
+AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
@@ -12,6 +13,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [weakdeps]
@@ -21,12 +23,14 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 PSBoardDataBaseInteractiveUtilsExt = "InteractiveUtils"
 
 [compat]
+AutoHashEquals = "2.2"
 CSV = "0.10"
 DBInterface = "2"
 DataFrames = "1"
 Documenter = "1"
 Downloads = "1"
 SQLite = "1"
+StaticArrays = "1.9"
 Tables = "1"
 
 [extras]

src/PSBoardDataBase.jl

@@ -9,6 +9,7 @@ using Dates
 
 include("parse_qaqc_master_log.jl")
 include("parse_clock.jl")
+include("SlaveLogParser.jl")
 
 include("create_table.jl")
 include("download_csv.jl")

src/SlaveLogParser.jl

@@ -0,0 +1,361 @@
+module SlaveLogParser
+
+using StaticArrays
+using AutoHashEquals
+
+const HEADER_QSPIP_START = "=============== Test QAPIp Start ==============="
+const HEADER_POWER_START = "=============== Test Power Start ==============="
+const HEADER_ASDTP_START = "=============== Test ASDTP Start ==============="
+const HEADER_RECOV_START = "=============== Test Recov Start ==============="
+HEADER_STARTS =
+    [HEADER_QSPIP_START, HEADER_POWER_START, HEADER_ASDTP_START, HEADER_RECOV_START]
+
+"""
+Indicate parser state.
+Default is `MODE_NONE`.
+In `MODE_NONE`, each line is feeded into parser to detect the start of each section.
+"""
+@enum SlaveLogSection begin
+    MODE_NONE
+    MODE_QSPIP
+    MODE_POWER
+    MODE_ASDTP
+    MODE_RECOV
+end
+
+struct PowerResult
+    result_3v3d::Float64
+    result_3v3a::Float64
+    result_n3va::Float64
+    fpga_temp::Float64
+
+    result::Bool
+end
+
+struct SlaveLogResult end
+
+"""
+    get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64,Int64,Bool}
+
+Extract info from slave log filename.
+"""
+function get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64, Int64, Bool}
+    main, _ext = splitext(filename)
+    parts = split(main, '_')
+    psbid = parse(Int64, parts[1])
+    runid = parse(Int64, parts[2])
+    islongrun = if length(parts) == 3
+        true
+    else
+        false
+    end
+
+    (psbid, runid, islongrun)
+end
+
+function is_valid_slavelog(filename::AbstractString)::Bool
+    error("not yet implemented")
+end
+
+"""
+    detect_mode_start(line::AbstractString)
+
+Detect [`SlaveLogSection`](@ref) from section starting header line.
+If the line doesn't match any section, returns `nothing`.
+"""
+function detect_mode_start(line::AbstractString)
+    if line == HEADER_QSPIP_START
+        MODE_QSPIP
+    elseif line == HEADER_POWER_START
+        MODE_POWER
+    elseif line == HEADER_ASDTP_START
+        MODE_ASDTP
+    elseif line == HEADER_RECOV_START
+        MODE_RECOV
+    else
+        nothing
+    end
+end
+
+"""
+    detect_mode_start!(mode::SlaveLogSection, line::AbstractString)
+
+Detect mode from the `line` and update `mode`.
+"""
+function detect_mode_start(mode::SlaveLogSection, line::AbstractString)
+    newmode = detect_mode_start(line)
+    if !isnothing(newmode)
+        mode = newmode
+    end
+    mode
+end
+
+"""
+    parse_qspip_section(lines::Base.Iterators.Stateful)
+
+Parse QSPIp section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+"""
+function parse_qspip_section!(lines::Base.Iterators.Stateful)
+    # TODO
+    nothing
+end
+
+"""
+    parse_power_section(lines::Base.Iterators.Stateful)
+
+Parse Power section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+"""
+function parse_power_section!(lines::Base.Iterators.Stateful)
+    # TODO
+    nothing
+end
+
+"""
+Measurement result for asic in asdtp test.
+"""
+@auto_hash_equals struct AsdtpMeasurement
+    before::Float64
+    current::Float64
+    next::Float64
+end
+
+AsdtpMeasurement(x::NTuple{3, <:Real}) = AsdtpMeasurement(x...)
+
+function Base.parse(::Type{AsdtpMeasurement}, s::AbstractString)
+    v = split(s, ':')
+    @assert length(v) == 3
+    AsdtpMeasurement(parse(Float64, v[1]), parse(Float64, v[2]), parse(Float64, v[3]))
+end
+
+@auto_hash_equals struct AsdtpResult
+    reconfig_done::Int64
+    always_hit_flag::Int64
+    autoreconfig::Union{Bool, Missing}
+    asdtp_main::Union{Vector{Vector{AsdtpMeasurement}}, Missing}
+    asdtp_reset::Int64
+    asdtp_total::Int64
+    reconfig_done_2::Int64
+    always_hit_flag_2::Int64
+    si_done::UInt64
+    lolb_in::UInt64
+    ppconfig_done::UInt64
+    ppconfig_error::UInt64
+    pllld_fail_counter::UInt64
+    ppconfig_fail_counter::UInt64
+    pp_pllds::MVector{8, UInt32}
+end
+
+"""
+    parse_asdtp_section(lines::Base.Iterators.Stateful)
+
+Parse ASDTP section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+"""
+function parse_asdtp_section!(lines::Base.Iterators.Stateful)
+    line = popfirst!(lines)
+    result_reconfig_done = let
+        m = match(r"^reconfig_done = (\d+)$", line)
+        parse(Int64, m[1])
+    end
+    line = popfirst!(lines)
+    result_always_hit_flag = let
+        m = match(r"^always_hit_flag = (\d+)$", line)
+        parse(Int64, m[1])
+    end
+    line = popfirst!(lines)
+    result_autoreconfig = if line == "Autoreconfig done"
+        true
+    elseif line == "Autoreconfig fail"
+        false
+    else
+        missing
+    end
+
+    result_asdtp_main = if !ismissing(result_autoreconfig) && result_autoreconfig
+        line_count = 0
+        line_count += 1
+        results = map(_ -> AsdtpMeasurement[], 1:8)
+        for asic_id in 1:8
+            header_line = "----PP$(asic_id)----"
+            while line != header_line
+                line = popfirst!(lines)
+                line_count += 1
+            end
+            for _ in 1:32
+                line = popfirst!(lines)
+                line_count += 1
+                mes = parse(AsdtpMeasurement, line)
+                push!(results[asic_id], mes)
+            end
+        end
+        @assert length(results[1]) == 32 "unexpected length: $(length(results[1]))"
+
+        line = popfirst!(lines)
+        @assert line == "100"
+        results
+    else
+        missing
+    end
+
+    line = popfirst!(lines)
+    result_asdtp_reset, result_asdtp_total = let
+        m = match(r"^ASDTP : (\d+) times reset : result = (\d+)$", line)
+        parse(Int64, m[1]), parse(Int64, m[2])
+    end
+    line = popfirst!(lines)
+    result_reconfig_done_2 = let
+        m = match(r"^reconfig_done = (\d+)$", line)
+        parse(Int64, m[1])
+    end
+    line = popfirst!(lines)
+    result_always_hit_flag_2 = let
+        m = match(r"^always_hit_flag = (\d+)$", line)
+        parse(Int64, m[1])
+    end
+    line = popfirst!(lines)
+    @assert line == ""
+
+    line = popfirst!(lines)
+    @assert line == "------- Done check -------" "actual line: $line"
+    line = popfirst!(lines)
+    result_si_done = let
+        m = match(r"^Si_done = (0x[[:xdigit:]]+)$", line)
+        parse(UInt64, m[1])
+    end
+    line = popfirst!(lines)
+    result_lolb_in = let
+        m = match(r"^LOLB_in = (0x[[:xdigit:]]+)$", line)
+        parse(UInt64, m[1])
+    end
+    line = popfirst!(lines)
+    result_ppconfig_done = let
+        m = match(r"^PPconfig_done = (0x[[:xdigit:]]+)$", line)
+        parse(UInt64, m[1])
+    end
+    line = popfirst!(lines)
+    result_ppconfig_error = let
+        m = match(r"^PPconfig_error = (0x[[:xdigit:]]+)$", line)
+        parse(UInt64, m[1])
+    end
+    line = popfirst!(lines)
+    result_pllld_fail_counter = let
+        m = match(r"^PLLLD_fail_counter = (0x[[:xdigit:]]+)$", line)
+        parse(UInt64, m[1])
+    end
+    line = popfirst!(lines)
+    result_ppconfig_fail_counter = let
+        m = match(r"^PPconfig_fail_counter = (0x[[:xdigit:]]+)$", line)
+        parse(UInt64, m[1])
+    end
+    result_pp_pllds = MVector{8, UInt32}(undef)
+    for ppid in 1:8
+        line = popfirst!(lines)
+        m = match(Regex("^PP$(ppid)_PLLLD = (0x[[:xdigit:]]+)\$"), line)
+        result_pp_pllds[ppid] = parse(UInt32, m[1])
+    end
+
+    AsdtpResult(
+        result_reconfig_done,
+        result_always_hit_flag,
+        result_autoreconfig,
+        result_asdtp_main,
+        result_asdtp_reset,
+        result_asdtp_total,
+        result_reconfig_done_2,
+        result_always_hit_flag_2,
+        result_si_done,
+        result_lolb_in,
+        result_ppconfig_done,
+        result_ppconfig_error,
+        result_pllld_fail_counter,
+        result_ppconfig_fail_counter,
+        result_pp_pllds,
+    )
+end
+
+"""
+    parse_qspip_section(lines::Base.Iterators.Stateful)
+
+Parse Recov section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+
+# Return
+- `nothing`: if failed to parse
+- `true`: if successed
+- `false`
+"""
+function parse_recov_section!(lines::Base.Iterators.Stateful)
+    line = popfirst!(lines)
+    if startswith("====")(line)
+        line = popfirst!(lines)
+    end
+    m = match(r"Test Recov Result = (\d+)", line)
+    if isnothing(m)
+        return nothing
+    else
+        return m[1] == "1"
+    end
+end
+
+"""
+    parse_slavelog_file(filename::AbstractString)
+
+Main function to parse slave log file.
+"""
+function parse_slavelog_file(filename::AbstractString)
+    lines_iter = Iterators.Stateful(eachline(filename))
+
+    asdtp_results = Any[]
+
+    mode::SlaveLogSection = MODE_NONE
+    # main loop
+    while !isempty(lines_iter)
+        # each sections
+        if mode == MODE_NONE
+            line = popfirst!(lines_iter)
+            mode = detect_mode_start(mode, line)
+        elseif mode == MODE_QSPIP
+            parse_qspip_section!(lines_iter)
+            mode = MODE_NONE
+        elseif mode == MODE_POWER
+            parse_power_section!(lines_iter)
+            mode = MODE_NONE
+        elseif mode == MODE_ASDTP
+            result = parse_asdtp_section!(lines_iter)
+            push!(asdtp_results, result)
+            mode = MODE_NONE
+        elseif mode == MODE_RECOV
+            parse_recov_section!(lines_iter)
+            mode = MODE_NONE
+        end
+    end
+
+    @info "Finished"
+    return asdtp_results
+end
+
+function eff99_count_map(asdtp_results)
+    # try(100) × channel(8) × channel(32)
+    @assert length(asdtp_results) == 100
+    @assert length(asdtp_results[begin]) == 8
+    @assert length(asdtp_results[begin][begin]) == 32
+    map(1:8) do i_asic
+        map(1:32) do i_channel
+            sum(1:100) do i_try
+                asdtp_results[i_try][i_asic][i_channel] != AsdtpMeasurement(0, 1, 0)
+            end
+        end
+    end
+end
+
+end

test/input/.gitignore

@@ -5,3 +5,6 @@ slavelogs/main/*
 !slavelogs/main/448_103_clk.txt
 !slavelogs/main/444_103_clk.txt
 !slavelogs/main/209_51_clk.txt
+!slavelogs/main/525_244.txt
+!slavelogs/main/525_245_longrun.txt
+!slavelogs/main/430_100.txt

test/runtests.jl

@@ -1,5 +1,6 @@
 using Test
 using PSBoardDataBase
+using StaticArrays
 using CSV, DataFrames
 using SQLite, DBInterface
 using Dates
@@ -45,6 +46,64 @@ true || include("../src/PSBoardDataBase.jl")
         ) == 22
     end
 
+    @testset "Slave Log parser" begin
+        lines = Iterators.Stateful(
+            Iterators.drop(eachline("./input/slavelogs/main/430_100.txt"), 2280),
+        )
+        result_asdtp = PSBoardDataBase.SlaveLogParser.parse_asdtp_section!(lines)
+        # target = PSBoardDataBase.SlaveLogParser.AsdtpResult(
+        #     1,
+        #     0,
+        #     true,
+        #     fill(PSBoardDataBase.SlaveLogParser.AsdtpMeasurement.(fill((0, 1, 0), 32)), 8),
+        #     0,
+        #     1,
+        #     1,
+        #     0,
+        #     0x1,
+        #     0x1,
+        #     0xff,
+        #     0x0,
+        #     0x0,
+        #     0x0,
+        #     MVector((0x1001, 0x2001, 0x3001, 0x4001, 0x5001, 0x6001, 0x7001, 0x8001)),
+        # )
+        # @info "" target == result_asdtp zip(result_asdtp.pp_pllds, target.pp_pllds) |>
+        #                                 collect result_asdtp.pp_pllds == target.pp_pllds Iterators.filter(
+        #     x -> x[1] != x[2],
+        #     zip(result_asdtp.pp_pllds, target.pp_pllds),
+        # ) |> collect
+        # @info "" map(fieldnames(typeof(result_asdtp))) do field
+        #     (; field, diff = getfield(result_asdtp, field) == getfield(target, field))
+        # end
+
+        @test result_asdtp == PSBoardDataBase.SlaveLogParser.AsdtpResult(
+            1,
+            0,
+            true,
+            fill(PSBoardDataBase.SlaveLogParser.AsdtpMeasurement.(fill((0, 1, 0), 32)), 8),
+            0,
+            1,
+            1,
+            0,
+            0x1,
+            0x1,
+            0xff,
+            0x0,
+            0x0,
+            0x0,
+            MVector((0x1001, 0x2001, 0x3001, 0x4001, 0x5001, 0x6001, 0x7001, 0x8001)),
+        )
+        @info "pass"
+        lines = Iterators.Stateful(
+            Iterators.drop(eachline("./input/slavelogs/main/525_244.txt"), 2280),
+        )
+        PSBoardDataBase.SlaveLogParser.parse_asdtp_section!(lines)
+        # PSBoardDataBase.SlaveLogParser.parse_slavelog_file(
+        #     "./input/slavelogs/main/525_244.txt",
+        # )
+    end
+
     @testset "Download data csv" begin
         out = tempname()
         @test CSV.read(