update: skew (skew calculating algorithm & analysis notebook)

examples/skew_multimeasure_stats.jl

@@ -22,6 +22,7 @@ begin
     using CairoMakie
     using Statistics
     using PlutoUI
+    using Random
 end
 
 # ╔═╡ f25e7e08-8a73-4cac-ac7c-d310725c558d
@@ -32,7 +33,7 @@ md"""
 """
 
 # ╔═╡ 1a6322d4-9deb-4709-aa2e-df7d8be5b16f
-TableOfContents()
+TableOfContents(depth = 4)
 
 # ╔═╡ 11537f91-e16b-45f0-9768-6df842371d36
 db = SQLite.DB("../psboard_qaqc.db")
@@ -81,6 +82,14 @@ qaqc_positions = DBInterface.execute(db, sql"select * from qaqc_positions") |> D
 # ╔═╡ 181c3fe6-d087-42e2-b175-3fb84c42e3e8
 position_id_skew_map = select(qaqc_positions, [:id, :rising_ns]) |> Tables.rowtable |> Dict
 
+# ╔═╡ dd669f14-989b-45ee-87d8-5d9cf282fafd
+md"""
+## Dispatch
+"""
+
+# ╔═╡ 1e9c3944-0cd4-40da-9014-a9153d4e95ed
+qaqc_dispatch = DBInterface.execute(db, sql"select * from qaqc_dispatch") |> DataFrame
+
 # ╔═╡ 322cb530-65a5-4973-86f8-01ccc2439cc4
 md"""
 # Clock Analysis
@@ -128,6 +137,9 @@ df_rawskews =
     ) |>
     DataFrame
 
+# ╔═╡ 0e680044-e4e1-4f39-a5c5-afa5c53fc7a7
+clk_files |> filter(contains("168")) .|> PSBoardDataBase.ClockParser.get_skew_and_riseup
+
 # ╔═╡ d7541b93-4c49-4dcd-bda0-91e447f44596
 # substract result of measurements of position dependency
 df_skews = let
@@ -142,8 +154,62 @@ df_skews = let
     select!(df, Not(:position))
 end
 
+# ╔═╡ e7faa647-79cd-4247-a3f2-9868c7b9d4ca
+md"""
+## skewの分布
+
+``skew = rising\_time_{psbid, position} - rising\_time_{position}``
+"""
+
+# ╔═╡ 3a412a98-4a2d-4bfb-8053-b0f480fae921
+let
+    skews = skipmissing(df_skews.skew) |> collect
+    npsbid = df_skews.psbid |> unique |> length
+    fig = Figure()
+    ax = Axis(
+        fig[1, 1],
+        title = "skews of all measurements (n_meas = $(length(skews)), n_psb = $(npsbid))",
+        xlabel = "skew / ns",
+    )
+    stephist!(ax, skews, bins = range(minimum(skews), maximum(skews), step = 1 / 57))
+    fig
+end
+
+# ╔═╡ b38bbed4-8721-4e92-a546-f7926cc07dd3
+md"""
+## 立ち上がり時間の分布
+"""
+
 # ╔═╡ 420dce0e-4757-48d9-84ec-7ddfac2fdff6
-stephist(df_skews.width |> skipmissing |> collect)
+let
+    skew_widths = df_skews.width |> skipmissing |> collect
+    hist(
+        skew_widths,
+        bins = range(minimum(skew_widths), maximum(skew_widths), step = 1 / 57),
+        bar_labels = :y,
+        label_size = 14,
+        label_formatter = x -> "$(round(Int, x))",
+        axis = (
+            title = "clock rise span distribution",
+            xlabel = "rise up span / ns",
+            ylabel = "counts",
+            limits = ((0, 0.18), (0, nothing)),
+        ),
+    )
+end
+
+# ╔═╡ 99902640-fee3-4502-9c7e-cb08834bad0b
+maximum(skipmissing(df_skews.width)) / (1 / 57)
+
+# ╔═╡ c79c6684-1b03-41b5-aa90-ef8c7a8eb69c
+md"""
+この結果を元に、クロック試験のしきい値は$(round(9 * 1 / 57; digits = 2))ns以上に設定
+"""
+
+# ╔═╡ ec774495-c0be-47a4-9d2c-b48159c07013
+md"""
+## 各PSBoardごとの統計
+"""
 
 # ╔═╡ d082e07c-3b42-4362-bebf-63356979a49b
 gdf_skews_on_psbid = groupby(df_skews, :psbid)
@@ -199,8 +265,11 @@ end
 save("clock_skew_stats.svg", fig_skew_stats)
 
 # ╔═╡ 79e2f5d8-4609-4e9f-949e-6dc1f88c0b19
-df_skew_stats_abnormals =
+df_skew_stats_abnormals = let
-    filter([:mean_skew, :std_skew] => ((m, s) -> m > -5 && s > 1), df_skew_stats)
+    df = filter([:mean_skew, :std_skew] => ((m, s) -> m > -5 && s > 1), df_skew_stats)
+    sort!(df, :psbid)
+    df
+end
 
 # ╔═╡ d607e10e-854f-4652-9a34-9e22a188e315
 let
@@ -224,19 +293,23 @@ end
 
 # ╔═╡ 2795fd06-2f59-4e5b-829d-a8e428646790
 md"""
-## 分散が異常に大きいやつ
+### 分散が異常に大きいやつ
 基本的に予想通り、分散は小さく複数回の測定で整合的な結果が得られているが、いくつか例外があった。
 
 はじめはpsbid 127(4回測定)が含まれていたが、これはデータベースの編集ミスであることがわかり、修正した結果、消えた。
 
-### psbid: 291
+!!! todo
+    他の例外も確認する。
+    `df_skew_stats_abnormals`を確認
+
+#### psbid: 291
 - run: 83, 94
 - 83でcommunication error(SFP半抜け)
-### psbid: 460
+#### psbid: 460
 - run: 105, 132
 - psbid 444と同じく電源の抜き差しによってクロックの0と1000が繰り返されたパターン
 - 追試に送られてる
-### psbid: 545 (**問題の**)
+#### psbid: 545 (**問題の**)
 - run: 126, 132
 - どちらも測定結果自体には変なところはない
   - どちらも1回だけ立ち上がりがある
@@ -244,6 +317,166 @@ md"""
 - 126が電源が不安定なときだったかもしれないが、記録がない
 """
 
+# ╔═╡ 26976b6c-3954-4a41-a99b-c1aaebdc645d
+md"""
+### skewの分布
+"""
+
+# ╔═╡ 38d472ca-6347-4096-828d-fd1256130a59
+df_skews_selected = combine(
+    gdf_skews_on_psbid,
+    sdf -> begin
+        all(ismissing, sdf.skew) && @info "" sdf
+        if nrow(sdf) == 1
+            @assert sdf.skew |> first |> !ismissing
+            (skew = sdf.skew |> first, width = sdf.width |> first)
+        else
+            if sdf.psbid[1] == 291
+                df = filter(:runid => ==(94), sdf)
+                @assert nrow(df) == 1
+                (skew = df.skew |> first, width = sdf.width |> first)
+            elseif sdf.psbid[1] == 460
+                df = filter(:runid => ==(132), sdf)
+                @assert nrow(df) == 1
+                (skew = df.skew |> first, width = sdf.width |> first)
+            elseif sdf.psbid[1] == 545
+                @error "temp"
+                df = filter(:runid => ==(132), sdf)
+                @assert nrow(df) == 1
+                (skew = df.skew |> first, width = sdf.width |> first)
+            else
+                # assume that runid is chronological
+                i = argmax(sdf.runid .|> (id -> ismissing(id) ? -1 : id))
+                (skew = sdf.skew[i], width = sdf.width[i])
+            end
+        end
+    end,
+)
+
+# ╔═╡ 310710da-ebb2-4f54-b238-38d493a6a533
+let
+    skews = df_skews_selected.skew |> skipmissing |> collect
+    fig = Figure()
+    ax = Axis(fig[1, 1], title = "skews (n = $(length(skews)))", xlabel = "skew / ns")
+    stephist!(ax, skews, bins = range(minimum(skews), maximum(skews), step = 1 / 57))
+    fig
+end
+
+# ╔═╡ e9964184-1a2a-4ab9-bc22-2705a25393ec
+filter(:skew => (x -> !ismissing(x) && x < -3), df_skews_selected)
+
+# ╔═╡ f2d0ea9b-ede7-496a-8d1f-9f748f9d1666
+transform(
+    filter(:skew => ismissing, df_skews_selected),
+    :psbid => ByRow(psbid -> begin
+        psbid in qaqc_dispatch.psb_id
+    end) => :is_dispatched,
+)
+
+# ╔═╡ f5bc8b1a-3fee-48ed-9910-367a1cda632e
+md"""
+- psbid: 255
+    - Si not locked
+"""
+
+# ╔═╡ 9f3f780d-da6e-44b4-a002-c2f088681e9c
+df_skews_selected_sorted = let
+    df = sort(df_skews_selected, :skew)
+    dropmissing!(df, :skew)
+    df
+end
+
+# ╔═╡ 6c3c7669-d594-425d-bf05-9aa217c7656e
+md"""
+## turn on curves
+- lines on
+    - the minimum and maximum skews
+    - the minimum and maximum widths
+"""
+
+# ╔═╡ e9b3f541-d87d-4424-8b82-be1b7b3273d8
+let
+    psbid_width_min, psbid_width_max, psbid_width_typical = let
+        df_normal = filter(:width => >(0.06), df_skews_selected_sorted)
+        sort!(df_normal, :width)
+        df_typical = filter(:width => (x -> 0.08 < x < 0.11), df_normal)
+        rng = Random.Xoshiro(123)
+        id_rand_typical = rand(rng, 1:nrow(df_typical))
+        first(df_normal).psbid, last(df_normal).psbid, df_typical[id_rand_typical, :psbid]
+    end
+    psbid_with_lines = [
+        psbid_width_max,
+        psbid_width_min,
+		psbid_width_typical,
+    ]
+
+    fig = Figure()
+    ax = Axis(
+        fig[1, 1],
+        title = "clock turn on curves",
+		subtitle = "aligned on the last count = 0 points",
+        xlabel = "ns",
+        ylabel = "counts",
+        limits = ((-0.05, 0.15), (0, 1000)),
+    )
+    for file in clk_files
+        psbid, runid = parse_filename(file)
+        if psbid == 255
+            @info "Si not locked: skipped" psbid
+            continue
+        end
+        single_runs = filter(
+            [:psboard_id, :runid] => (
+                (ref_psbid, ref_runid) -> begin
+                    psbid == ref_psbid && runid == ref_runid
+                end
+            ),
+            qaqc_single_results,
+        )
+        if nrow(single_runs) != 1
+            if nrow(single_runs) != 0
+                @info "skipped" psbid runid nrow(single_runs)
+            end
+            continue
+        end
+        # offset = qaqc_positions.rising_ns[single_runs.position[1]]
+        offset_pos = let
+            # df = filter(:psbid => ==(psbid), df_skews_selected_sorted)
+            # @assert nrow(df) == 1 "nrow: $(nrow(df)), psbid: $(psbid)"
+            # df.skew[1] + 
+			qaqc_positions.rising_ns[single_runs.position[1]]
+        end
+        if psbid in psbid_with_lines
+            points =
+                eachline(file) .|>
+                PSBoardDataBase.ClockParser._parse_line .|>
+                (x -> (x[1] - offset_pos, x[2]))
+			id_first_over0 = findfirst(points) do ((time, count))
+				count > 0
+			end
+			@info "" points[id_first_over0] points[id_first_over0 - 1]
+			points = map(points) do ((time, count))
+				time - points[id_first_over0][1], count
+			end
+			stds = map(points) do ((time, count))
+				sqrt(count * (1000 - count) / 1000)
+			end
+            label, color = if psbid == psbid_width_max
+                "lomgest ($psbid)", (:red, 0.8)
+            elseif psbid == psbid_width_min
+                "shortest ($psbid)", (:blue, 0.8)
+			elseif psbid == psbid_width_typical
+				"typical ($psbid)", (:green, 0.8)
+            end
+            scatterlines!(ax, points, label = label, color = color)
+			@info "" first.(points) stds
+			errorbars!(ax, points, stds, whiskerwidth = 10, color = color)
+        end
+    end
+    axislegend(ax, position = :rb)
+    fig
+end
+
 # ╔═╡ Cell order:
 # ╟─f25e7e08-8a73-4cac-ac7c-d310725c558d
 # ╠═7c69d12c-80a5-11ef-2674-e155a3483342
@@ -260,13 +493,22 @@ md"""
 # ╟─33e099bc-ac4b-4b5f-88e7-20f4463c98ef
 # ╠═0e13f848-0efb-4775-9e3e-518b32588a79
 # ╠═181c3fe6-d087-42e2-b175-3fb84c42e3e8
+# ╟─dd669f14-989b-45ee-87d8-5d9cf282fafd
+# ╠═1e9c3944-0cd4-40da-9014-a9153d4e95ed
 # ╟─322cb530-65a5-4973-86f8-01ccc2439cc4
 # ╠═c1caca5f-4cfd-4f22-82b4-7925002359e6
 # ╠═3e5607fd-2a8a-4a1a-9e7b-3f23ef216fad
 # ╠═c1b9c0c3-00f8-4199-b07f-8888f1be625c
 # ╠═d6d04013-e0e4-49d5-a450-07ae164bfaa3
+# ╠═0e680044-e4e1-4f39-a5c5-afa5c53fc7a7
 # ╠═d7541b93-4c49-4dcd-bda0-91e447f44596
+# ╟─e7faa647-79cd-4247-a3f2-9868c7b9d4ca
+# ╠═3a412a98-4a2d-4bfb-8053-b0f480fae921
+# ╟─b38bbed4-8721-4e92-a546-f7926cc07dd3
 # ╠═420dce0e-4757-48d9-84ec-7ddfac2fdff6
+# ╠═99902640-fee3-4502-9c7e-cb08834bad0b
+# ╠═c79c6684-1b03-41b5-aa90-ef8c7a8eb69c
+# ╟─ec774495-c0be-47a4-9d2c-b48159c07013
 # ╠═d082e07c-3b42-4362-bebf-63356979a49b
 # ╠═25688d24-5aee-43d3-aff9-b9efa0556070
 # ╠═239a808c-0411-4542-ae68-6ae6af333bd2
@@ -277,3 +519,12 @@ md"""
 # ╠═79e2f5d8-4609-4e9f-949e-6dc1f88c0b19
 # ╠═d607e10e-854f-4652-9a34-9e22a188e315
 # ╠═2795fd06-2f59-4e5b-829d-a8e428646790
+# ╟─26976b6c-3954-4a41-a99b-c1aaebdc645d
+# ╠═38d472ca-6347-4096-828d-fd1256130a59
+# ╠═310710da-ebb2-4f54-b238-38d493a6a533
+# ╠═e9964184-1a2a-4ab9-bc22-2705a25393ec
+# ╠═f2d0ea9b-ede7-496a-8d1f-9f748f9d1666
+# ╠═f5bc8b1a-3fee-48ed-9910-367a1cda632e
+# ╠═9f3f780d-da6e-44b4-a002-c2f088681e9c
+# ╠═6c3c7669-d594-425d-bf05-9aa217c7656e
+# ╠═e9b3f541-d87d-4424-8b82-be1b7b3273d8

src/parse_clock.jl

@@ -22,19 +22,21 @@ function get_skew(file::T) where {T <: AbstractString}
     @debug "file: $(file)"
     lines = Iterators.Stateful(eachline(file))
 
-    first_line = popfirst!(lines)
+    was_0_before = false
-    # criteria: https://gitlab.cern.ch/dhashimo/PS_Board_QAQC/-/blob/master/Software_Test/project_auto_reset/QAQC_test_func.cxx?ref_type=heads#L208
+    # criteria is changed from QAQC for skew parsing
     let
+        first_line = popfirst!(lines)
         time, high = _parse_line(first_line)
-        if high > 0
+        if high == 0
-            @debug "Unexpected first line" file
+            was_0_before = true
-            return missing
         end
     end
 
     time_and_counts = Iterators.map(_parse_line, lines)
     for (time, high) in time_and_counts
-        if high >= 500
+        if high == 0
+            was_0_before = true
+        elseif was_0_before && high >= 500
             return time
         end
     end
@@ -93,17 +95,14 @@ this returns Tuple of 2 `missing`s.
 function get_skew_and_riseup(file::T) where {T <: AbstractString}
     lines = Iterators.Stateful(eachline(file))
 
-    last_low_time = 0.0
+    last_low_time = missing
-    first_high_time = 0.0
+    first_high_time = missing
-    skew = 0.0
+    skew = missing
     is_rised = false
 
     let
         _time, high = _parse_line(popfirst!(lines))
-        if high > 0
+        if high == 0
-            @debug "Unexpected first line"
-            return (missing, missing)
-        elseif high == 0
             last_low_time = time
         end
     end
@@ -112,16 +111,16 @@ function get_skew_and_riseup(file::T) where {T <: AbstractString}
         time, high = _parse_line(line)
         if high == 0
             last_low_time = time
-        elseif !is_rised && high >= 500
+        elseif !ismissing(last_low_time) && !is_rised && high >= 500
             skew = time
             is_rised = true
-        elseif high == 1000
+        elseif !ismissing(skew) && high == 1000
             first_high_time = time
             break
         end
     end
 
-    if first_high_time == 0
+    if first_high_time === missing
         @debug "Clock skew out of range"
         return (missing, missing)
     end

test/input/.gitignore

@@ -4,3 +4,4 @@ slavelogs/main/*
 !slavelogs/main/230_51_clk.txt
 !slavelogs/main/448_103_clk.txt
 !slavelogs/main/444_103_clk.txt
+!slavelogs/main/209_51_clk.txt

test/runtests.jl

@@ -18,14 +18,16 @@ true || include("../src/PSBoardDataBase.jl")
     end
 
     @testset "parse clk log" begin
-        @test PSBoardDataBase.ClockParser.get_skew("input/slavelogs/main/230_51_clk.txt") |>
+        @test PSBoardDataBase.ClockParser.get_skew("input/slavelogs/main/230_51_clk.txt") ==
-              ismissing
+              14.178571428571429
         @test PSBoardDataBase.ClockParser.get_skew("input/slavelogs/main/448_103_clk.txt") ≈
               12.000000000000000
+        @test PSBoardDataBase.ClockParser.get_skew("input/slavelogs/main/209_51_clk.txt") ==
+              12.571428571428571
 
         @test PSBoardDataBase.ClockParser.get_skew_and_riseup(
             "input/slavelogs/main/230_51_clk.txt",
-        ) === (missing, missing)
+        ) === (14.178571428571429, 14.196428571428571 - 14.107142857142858)
         @test PSBoardDataBase.ClockParser.get_skew_and_riseup(
             "input/slavelogs/main/448_103_clk.txt",
         ) == (12.0, 12.053571428571429 - 11.982142857142858)