commit 07971df5e4ed31798efc0fc4b42abb317f976d32
Author: Wataru Otsubo <wotsubo@icepp.s.u-tokyo.ac.jp>
Date:   Sat Jul 27 17:58:35 2024 +0900

    init: partial slave log parser

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..22cfd68
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+./slavelogs/
+./ps_all.csv
+./masterlogs/
diff --git a/analysis.jl b/analysis.jl
new file mode 100644
index 0000000..97e9c6b
--- /dev/null
+++ b/analysis.jl
@@ -0,0 +1,68 @@
+using CSV
+using DataFrames
+using GLMakie
+
+"""
+mapping to unique numbers
+"""
+function unique_num(v)
+    uniques = unique(v)
+    mapping = Dict(uniques .=> eachindex(uniques))
+    map(v) do x
+        mapping[x]
+    end
+end
+
+df = CSV.read("ps_all.csv", DataFrame)
+dropmissing!(df, :position)
+
+gdf = groupby(df, :position)
+
+df_stacked = let
+    preheaders = [:qspip, :recov, :power, :clock]
+    postheaders = map(preheaders) do sym
+        Symbol("n" * string(sym))
+    end
+    df = combine(
+        gdf,
+        preheaders .=> (v -> count(v .!= 1)) .=> postheaders,
+    )
+
+    stack(df, postheaders)
+end
+
+transform!(
+    df_stacked,
+    :position => unique_num => :position_id,
+)
+transform!(
+    df_stacked,
+    :variable => unique_num => :variable_id,
+)
+
+colors = Makie.wong_colors()
+fig = Figure()
+ax = Axis(
+    fig[1, 1],
+    title = "failed ones",
+    xlabel = "position",
+    xticks = (df_stacked.position_id, df_stacked.position),
+    xticklabelrotation = π / 3,
+    ylabel = "counts",
+)
+barplot!(
+    ax,
+    df_stacked.position_id,
+    df_stacked.value,
+    stack = df_stacked.variable_id,
+    color = colors[df_stacked.variable_id |> collect],
+)
+
+Legend(
+    fig[1, 2],
+    [PolyElement(polycolor = colors[i]) for i in df_stacked.variable_id |> unique],
+    df_stacked.variable |> unique,
+    "entries",
+)
+
+save("failed_reasons.png", fig)
diff --git a/failed_reasons.png b/failed_reasons.png
new file mode 100644
index 0000000..60c845c
Binary files /dev/null and b/failed_reasons.png differ
diff --git a/slavelog_parser.jl b/slavelog_parser.jl
new file mode 100644
index 0000000..c9dbba1
--- /dev/null
+++ b/slavelog_parser.jl
@@ -0,0 +1,207 @@
+HEADER_QSPIP_START = "=============== Test QAPIp Start ==============="
+HEADER_POWER_START = "=============== Test Power Start ==============="
+HEADER_ASDTP_START = "=============== Test ASDTP Start ==============="
+HEADER_RECOV_START = "=============== Test Recov Start ==============="
+HEADER_STARTS = [HEADER_QSPIP_START, HEADER_POWER_START, HEADER_ASDTP_START, HEADER_RECOV_START]
+
+@enum SlaveLogSection begin
+    MODE_NONE
+    MODE_QSPIP
+    MODE_POWER
+    MODE_ASDTP
+    MODE_RECOV
+end
+
+"""
+Extract info from slave log filename.
+"""
+function get_psbid_runid_from_filename(filename::AbstractString)::Tuple{Int64,Int64,Bool}
+    main, _ext = splitext(filename)
+    parts = split(main, '_')
+    psbid = parse(Int64, parts[1])
+    runid = parse(Int64, parts[2])
+    islongrun = if length(parts) == 3
+        true
+    else
+        false
+    end
+
+    (psbid, runid, islongrun)
+end
+
+function is_valid_slavelog(filename::AbstractString)::Bool
+    error("not yet implemented")
+end
+
+"""
+    detect_mode_start(line::AbstractString)
+
+Detect [`SlaveLogSection`](@ref) from section starting header line.
+If the line doesn't match any section, returns `nothing`.
+"""
+function detect_mode_start(line::AbstractString)
+    if line == HEADER_QSPIP_START
+        MODE_QSPIP
+    elseif line == HEADER_POWER_START
+        MODE_POWER
+    elseif line == HEADER_ASDTP_START
+        MODE_ASDTP
+    elseif line == HEADER_RECOV_START
+        MODE_RECOV
+    else
+        nothing
+    end
+end
+
+"""
+    detect_mode_start!(mode::SlaveLogSection, line::AbstractString)
+
+Detect mode from the `line` and update `mode`.
+"""
+function detect_mode_start(mode::SlaveLogSection, line::AbstractString)
+    newmode = detect_mode_start(line)
+    if !isnothing(newmode)
+        mode = newmode
+    end
+    mode
+end
+
+"""
+    parse_qspip_section(lines::Base.Iterators.Stateful)
+
+Parse QSPIp section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+"""
+function parse_qspip_section!(lines::Base.Iterators.Stateful)
+    # TODO
+    nothing
+end
+
+"""
+    parse_power_section(lines::Base.Iterators.Stateful)
+
+Parse Power section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+"""
+function parse_power_section!(lines::Base.Iterators.Stateful)
+    # TODO
+    nothing
+end
+
+"""
+Measurement result for asic in asdtp test.
+"""
+struct AsdtpMeasurement
+    before::Float64
+    current::Float64
+    next::Float64
+end
+
+function Base.parse(::Type{AsdtpMeasurement}, s::AbstractString)
+    v = split(s, ':')
+    @assert length(v) == 3
+    AsdtpMeasurement(
+        parse(Float64, v[1]),
+        parse(Float64, v[2]),
+        parse(Float64, v[3]),
+    )
+end
+
+"""
+    parse_asdtp_section(lines::Base.Iterators.Stateful)
+
+Parse ASDTP section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+"""
+function parse_asdtp_section!(lines::Base.Iterators.Stateful)
+    line_count = 0
+    line = popfirst!(lines)
+    line_count += 1
+    results = map(_ -> AsdtpMeasurement[], 1:8)
+    for asic_id in 1:8
+        header_line = "----PP$(asic_id)----"
+        while line != header_line
+            line = popfirst!(lines)
+            line_count += 1
+        end
+        for _ in 1:32
+            line = popfirst!(lines)
+            line_count += 1
+            mes = parse(AsdtpMeasurement, line)
+            push!(results[asic_id], mes)
+        end
+    end
+    @assert length(results[1]) == 32 "unexpected length: $(length(results[1]))"
+    return results
+end
+
+"""
+    parse_qspip_section(lines::Base.Iterators.Stateful)
+
+Parse Recov section of given stateful iterator of log.
+
+# Args
+- `lines`: Stateful iterator of slave log file lines
+"""
+function parse_recov_section!(lines::Base.Iterators.Stateful)
+    # TODO
+    nothing
+end
+
+function parse_slavelog_file(filename::AbstractString)
+    lines_iter = Iterators.Stateful(eachline(filename))
+
+    asdtp_results = Any[]
+
+    mode::SlaveLogSection = MODE_NONE
+    # main loop
+    while !isempty(lines_iter)
+        # each sections
+        if mode == MODE_NONE
+            line = popfirst!(lines_iter)
+            mode = detect_mode_start(mode, line)
+        elseif mode == MODE_QSPIP
+            parse_qspip_section!(lines_iter)
+            mode = MODE_NONE
+        elseif mode == MODE_POWER
+            parse_power_section!(lines_iter)
+            mode = MODE_NONE
+        elseif mode == MODE_ASDTP
+            result = parse_asdtp_section!(lines_iter)
+            push!(asdtp_results, result)
+            mode = MODE_NONE
+        elseif mode == MODE_RECOV
+            parse_recov_section!(lines_iter)
+            mode = MODE_NONE
+        end
+    end
+
+    @info "Finished"
+    return asdtp_results
+end
+
+function eff99_count_map(asdtp_results)
+    # try(100) × channel(8) × channel(32)
+    @assert length(asdtp_results) == 100
+    @assert length(asdtp_results[begin]) == 8
+    @assert length(asdtp_results[begin][begin]) == 32
+    map(1:8) do i_asic
+        map(1:32) do i_channel
+            sum(1:100) do i_try
+                asdtp_results[i_try][i_asic][i_channel] != AsdtpMeasurement(0, 1, 0)
+            end
+        end
+    end
+end
+
+map(eachrow(filter(:position => ==("B-0-1"), df))) do row
+    readdir("slavelogs/") |>
+    filter(startswith("$(row.motherboard_id)_")) |>
+    filter(contains("longrun"))
+end |> Iterators.flatten