research/PSBoardDataBase
1
0
Fork 0
mirror of https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git synced 2025-07-02 17:49:28 +09:00
Code Issues Projects Releases Packages Wiki Activity Actions

rename: skew_multimeasure_stats.jl -> skew_stats.jl

Browse source
This commit is contained in:
Wataru Otsubo 2024-10-04 20:00:58 +09:00
parent a040d8fd33
commit ce5931a928
2 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

530
examples/skew_stats.jl Normal file
View file

@ -0,0 +1,530 @@
### A Pluto.jl notebook ###
# v0.19.46
using Markdown
using InteractiveUtils
# ╔═╡ 7c69d12c-80a5-11ef-2674-e155a3483342
begin
using Pkg
Pkg.activate("..")
Pkg.status()
true || include("../src/PSBoardDataBase.jl")
end
# ╔═╡ effa7ed9-2ac4-4468-a474-e2bb662580fe
begin
using PSBoardDataBase
using SQLite
using DataFrames
using DBInterface
using Tables
using CairoMakie
using Statistics
using PlutoUI
using Random
end
# ╔═╡ f25e7e08-8a73-4cac-ac7c-d310725c558d
md"""
# Detailed research on clock skew measurement
- distribution of clock skews
- consistency of multiple measurement on the same board
"""
# ╔═╡ 1a6322d4-9deb-4709-aa2e-df7d8be5b16f
TableOfContents(depth = 4)
# ╔═╡ 11537f91-e16b-45f0-9768-6df842371d36
db = SQLite.DB("../psboard_qaqc.db")
# ╔═╡ 268d3015-b8d3-48d9-b74a-062e258e0ec1
SQLite.tables(db)
# ╔═╡ 62105832-df1f-4834-8da6-c542e22207d1
md"""
## Single runs
"""
# ╔═╡ dea6f143-7916-4765-92f6-2bfb97a72835
qaqc_single_results =
DBInterface.execute(db, sql"select * from qaqc_single_run_results") |> DataFrame
# ╔═╡ 41543c0c-d7c4-447b-a268-0d356c88d92c
md"""
## PS Board list
"""
# ╔═╡ 633ecdee-7e2f-4de8-833a-21cd0351c1f1
DBInterface.execute(
db,
sql"""
select * from ps_boards
""",
) |> DataFrame
# ╔═╡ 87f1966e-3b07-4f9d-8fc4-7b7aa4319d50
md"""
## Run list
"""
# ╔═╡ f379d43c-9300-41f4-b0fc-3c9d749e3105
qaqc_runs = DBInterface.execute(db, sql"select * from qaqc_runs") |> DataFrame
# ╔═╡ 33e099bc-ac4b-4b5f-88e7-20f4463c98ef
md"""
## Positions
"""
# ╔═╡ 0e13f848-0efb-4775-9e3e-518b32588a79
qaqc_positions = DBInterface.execute(db, sql"select * from qaqc_positions") |> DataFrame
# ╔═╡ 181c3fe6-d087-42e2-b175-3fb84c42e3e8
position_id_skew_map = select(qaqc_positions, [:id, :rising_ns]) |> Tables.rowtable |> Dict
# ╔═╡ dd669f14-989b-45ee-87d8-5d9cf282fafd
md"""
## Dispatch
"""
# ╔═╡ 1e9c3944-0cd4-40da-9014-a9153d4e95ed
qaqc_dispatch = DBInterface.execute(db, sql"select * from qaqc_dispatch") |> DataFrame
# ╔═╡ 322cb530-65a5-4973-86f8-01ccc2439cc4
md"""
# Clock Analysis
main part
"""
# ╔═╡ c1caca5f-4cfd-4f22-82b4-7925002359e6
clk_files =
readdir("../test/input/slavelogs/main/", join = true) |>
filter(endswith("_clk.txt")) |>
filter(!contains("nagoya")) |>
filter(!contains("630_190"))
# ╔═╡ 3e5607fd-2a8a-4a1a-9e7b-3f23ef216fad
"""
Get `(psbid, runid)`.
"""
function parse_filename(filename::AbstractString)
m = match(r"(?<psbid>\d+)_(?<runid>\d+)_clk\.txt", filename)
parse(Int64, m[:psbid]), parse(Int64, m[:runid])
end
# ╔═╡ c1b9c0c3-00f8-4199-b07f-8888f1be625c
parse_filename("190_23_clk.txt")
# ╔═╡ d6d04013-e0e4-49d5-a450-07ae164bfaa3
# Get skew and rise up time from clock measurement files
# Use measurements recorded in qaqc_single_results
df_rawskews =
clk_files .|>
(
file -> begin
skew_width = PSBoardDataBase.ClockParser.get_skew_and_riseup(file)
psbid, runid = parse_filename(file)
(psbid = psbid, runid = runid, skew = skew_width[1], width = skew_width[2])
end
) |>
filter(
x ->
filter(
[:psboard_id, :runid] =>
((psbid, runid) -> (psbid == x.psbid && runid == x.runid)),
qaqc_single_results,
) |> !isempty,
) |>
DataFrame
# ╔═╡ 0e680044-e4e1-4f39-a5c5-afa5c53fc7a7
clk_files |> filter(contains("168")) .|> PSBoardDataBase.ClockParser.get_skew_and_riseup
# ╔═╡ d7541b93-4c49-4dcd-bda0-91e447f44596
# substract result of measurements of position dependency
df_skews = let
df = leftjoin(
df_rawskews,
@view(qaqc_single_results[!, [:psboard_id, :runid, :position]]),
on = [:psbid => :psboard_id, :runid],
)
leftjoin!(df, @view(qaqc_positions[!, [:id, :rising_ns]]), on = [:position => :id])
transform!(df, [:skew, :rising_ns] => ByRow((x, y) -> x - y) => :skew)
select!(df, Not(:rising_ns))
select!(df, Not(:position))
end
# ╔═╡ e7faa647-79cd-4247-a3f2-9868c7b9d4ca
md"""
## skewの分布
``skew = rising\_time_{psbid, position} - rising\_time_{position}``
"""
# ╔═╡ 3a412a98-4a2d-4bfb-8053-b0f480fae921
let
skews = skipmissing(df_skews.skew) |> collect
npsbid = df_skews.psbid |> unique |> length
fig = Figure()
ax = Axis(
fig[1, 1],
title = "skews of all measurements (n_meas = $(length(skews)), n_psb = $(npsbid))",
xlabel = "skew / ns",
)
stephist!(ax, skews, bins = range(minimum(skews), maximum(skews), step = 1 / 57))
fig
end
# ╔═╡ b38bbed4-8721-4e92-a546-f7926cc07dd3
md"""
## 立ち上がり時間の分布
"""
# ╔═╡ 420dce0e-4757-48d9-84ec-7ddfac2fdff6
let
skew_widths = df_skews.width |> skipmissing |> collect
hist(
skew_widths,
bins = range(minimum(skew_widths), maximum(skew_widths), step = 1 / 57),
bar_labels = :y,
label_size = 14,
label_formatter = x -> "$(round(Int, x))",
axis = (
title = "clock rise span distribution",
xlabel = "rise up span / ns",
ylabel = "counts",
limits = ((0, 0.18), (0, nothing)),
),
)
end
# ╔═╡ 99902640-fee3-4502-9c7e-cb08834bad0b
maximum(skipmissing(df_skews.width)) / (1 / 57)
# ╔═╡ c79c6684-1b03-41b5-aa90-ef8c7a8eb69c
md"""
この結果を元に、クロック試験のしきい値は$(round(9 * 1 / 57; digits = 2))ns以上に設定
"""
# ╔═╡ ec774495-c0be-47a4-9d2c-b48159c07013
md"""
## 各PSBoardごとの統計
"""
# ╔═╡ d082e07c-3b42-4362-bebf-63356979a49b
gdf_skews_on_psbid = groupby(df_skews, :psbid)
# ╔═╡ 25688d24-5aee-43d3-aff9-b9efa0556070
combine(nrow, gdf_skews_on_psbid)
# ╔═╡ 239a808c-0411-4542-ae68-6ae6af333bd2
df_nrow_ordered = let
df = combine(nrow, gdf_skews_on_psbid)
sort!(df, :nrow, rev = true)
end
# ╔═╡ 8e57bde1-5f97-483d-906e-8ebfb65016d0
@view(df_nrow_ordered[findall(>(1), df_nrow_ordered.nrow), :])
# ╔═╡ 92c2ac3f-8034-4e9e-aadb-8bb166fbc948
df_skew_stats = let
df = combine(
gdf_skews_on_psbid,
sdf -> begin
if nrow(sdf) == 1
(; mean_skew = mean(sdf.skew), std_skew = missing, n = 1)
else
(; mean_skew = mean(sdf.skew), std_skew = std(sdf.skew), n = nrow(sdf))
end
end,
)
dropmissing!(df)
df
end
# ╔═╡ 893253c3-f0b2-401f-b892-b23291bcf5c1
fig_skew_stats = let
fig, ax, sc = scatter(
df_skew_stats.mean_skew,
df_skew_stats.std_skew,
marker = :x,
color = (Makie.wong_colors()[1], 0.8),
axis = (title = "skew mean vs std", xlabel = "mean", ylabel = "std"),
)
text!(
ax,
df_skew_stats.mean_skew,
df_skew_stats.std_skew,
text = string.(df_skew_stats.psbid),
color = (:gray, 0.5),
)
fig
end
# ╔═╡ 6467dcaa-6bd6-45c7-8c08-b310a09b8b0b
save("clock_skew_stats.svg", fig_skew_stats)
# ╔═╡ 79e2f5d8-4609-4e9f-949e-6dc1f88c0b19
df_skew_stats_abnormals = let
df = filter([:mean_skew, :std_skew] => ((m, s) -> m > -5 && s > 1), df_skew_stats)
sort!(df, :psbid)
df
end
# ╔═╡ d607e10e-854f-4652-9a34-9e22a188e315
let
df = df_skew_stats_abnormals
fig, ax, sc = scatter(
df.mean_skew,
df.std_skew,
marker = :x,
color = (Makie.wong_colors()[1], 0.8),
axis = (title = "skew mean vs std", xlabel = "mean", ylabel = "std"),
)
text!(
ax,
df.mean_skew,
df.std_skew,
text = string.(df.psbid) .* "," .* string.(df.n),
color = (:gray, 0.7),
)
fig
end
# ╔═╡ 2795fd06-2f59-4e5b-829d-a8e428646790
md"""
### 分散が異常に大きいやつ
基本的に予想通り、分散は小さく複数回の測定で整合的な結果が得られているが、いくつか例外があった。
はじめはpsbid 127(4回測定)が含まれていたが、これはデータベースの編集ミスであることがわかり、修正した結果、消えた。
!!! todo
他の例外も確認する。
`df_skew_stats_abnormals`を確認
#### psbid: 291
- run: 83, 94
- 83でcommunication error(SFP半抜け)
#### psbid: 460
- run: 105, 132
- psbid 444と同じく電源の抜き差しによってクロックの0と1000が繰り返されたパターン
- 追試に送られてる
#### psbid: 545 (**問題の**)
- run: 126, 132
- どちらも測定結果自体には変なところはない
- どちらも1回だけ立ち上がりがある
- 立ち上がりもそれほど長くない
- 126が電源が不安定なときだったかもしれないが、記録がない
"""
# ╔═╡ 26976b6c-3954-4a41-a99b-c1aaebdc645d
md"""
### skewの分布
"""
# ╔═╡ 38d472ca-6347-4096-828d-fd1256130a59
df_skews_selected = combine(
gdf_skews_on_psbid,
sdf -> begin
all(ismissing, sdf.skew) && @info "" sdf
if nrow(sdf) == 1
@assert sdf.skew |> first |> !ismissing
(skew = sdf.skew |> first, width = sdf.width |> first)
else
if sdf.psbid[1] == 291
df = filter(:runid => ==(94), sdf)
@assert nrow(df) == 1
(skew = df.skew |> first, width = sdf.width |> first)
elseif sdf.psbid[1] == 460
df = filter(:runid => ==(132), sdf)
@assert nrow(df) == 1
(skew = df.skew |> first, width = sdf.width |> first)
elseif sdf.psbid[1] == 545
@error "temp"
df = filter(:runid => ==(132), sdf)
@assert nrow(df) == 1
(skew = df.skew |> first, width = sdf.width |> first)
else
# assume that runid is chronological
i = argmax(sdf.runid .|> (id -> ismissing(id) ? -1 : id))
(skew = sdf.skew[i], width = sdf.width[i])
end
end
end,
)
# ╔═╡ 310710da-ebb2-4f54-b238-38d493a6a533
let
skews = df_skews_selected.skew |> skipmissing |> collect
fig = Figure()
ax = Axis(fig[1, 1], title = "skews (n = $(length(skews)))", xlabel = "skew / ns")
stephist!(ax, skews, bins = range(minimum(skews), maximum(skews), step = 1 / 57))
fig
end
# ╔═╡ e9964184-1a2a-4ab9-bc22-2705a25393ec
filter(:skew => (x -> !ismissing(x) && x < -3), df_skews_selected)
# ╔═╡ f2d0ea9b-ede7-496a-8d1f-9f748f9d1666
transform(
filter(:skew => ismissing, df_skews_selected),
:psbid => ByRow(psbid -> begin
psbid in qaqc_dispatch.psb_id
end) => :is_dispatched,
)
# ╔═╡ f5bc8b1a-3fee-48ed-9910-367a1cda632e
md"""
- psbid: 255
- Si not locked
"""
# ╔═╡ 9f3f780d-da6e-44b4-a002-c2f088681e9c
df_skews_selected_sorted = let
df = sort(df_skews_selected, :skew)
dropmissing!(df, :skew)
df
end
# ╔═╡ 6c3c7669-d594-425d-bf05-9aa217c7656e
md"""
## turn on curves
- lines on
- the minimum and maximum skews
- the minimum and maximum widths
"""
# ╔═╡ e9b3f541-d87d-4424-8b82-be1b7b3273d8
let
psbid_width_min, psbid_width_max, psbid_width_typical = let
df_normal = filter(:width => >(0.06), df_skews_selected_sorted)
sort!(df_normal, :width)
df_typical = filter(:width => (x -> 0.08 < x < 0.11), df_normal)
rng = Random.Xoshiro(123)
id_rand_typical = rand(rng, 1:nrow(df_typical))
first(df_normal).psbid, last(df_normal).psbid, df_typical[id_rand_typical, :psbid]
end
psbid_with_lines = [
psbid_width_max,
psbid_width_min,
psbid_width_typical,
]
fig = Figure()
ax = Axis(
fig[1, 1],
title = "clock turn on curves",
subtitle = "aligned on the last count = 0 points",
xlabel = "ns",
ylabel = "counts",
limits = ((-0.05, 0.15), (0, 1000)),
)
for file in clk_files
psbid, runid = parse_filename(file)
if psbid == 255
@info "Si not locked: skipped" psbid
continue
end
single_runs = filter(
[:psboard_id, :runid] => (
(ref_psbid, ref_runid) -> begin
psbid == ref_psbid && runid == ref_runid
end
),
qaqc_single_results,
)
if nrow(single_runs) != 1
if nrow(single_runs) != 0
@info "skipped" psbid runid nrow(single_runs)
end
continue
end
# offset = qaqc_positions.rising_ns[single_runs.position[1]]
offset_pos = let
# df = filter(:psbid => ==(psbid), df_skews_selected_sorted)
# @assert nrow(df) == 1 "nrow: $(nrow(df)), psbid: $(psbid)"
# df.skew[1] +
qaqc_positions.rising_ns[single_runs.position[1]]
end
if psbid in psbid_with_lines
points =
eachline(file) .|>
PSBoardDataBase.ClockParser._parse_line .|>
(x -> (x[1] - offset_pos, x[2]))
id_first_over0 = findfirst(points) do ((time, count))
count > 0
end
@info "" points[id_first_over0] points[id_first_over0 - 1]
points = map(points) do ((time, count))
time - points[id_first_over0][1], count
end
stds = map(points) do ((time, count))
sqrt(count * (1000 - count) / 1000)
end
label, color = if psbid == psbid_width_max
"lomgest ($psbid)", (:red, 0.8)
elseif psbid == psbid_width_min
"shortest ($psbid)", (:blue, 0.8)
elseif psbid == psbid_width_typical
"typical ($psbid)", (:green, 0.8)
end
scatterlines!(ax, points, label = label, color = color)
@info "" first.(points) stds
errorbars!(ax, points, stds, whiskerwidth = 10, color = color)
end
end
axislegend(ax, position = :rb)
fig
end
# ╔═╡ Cell order:
# ╟─f25e7e08-8a73-4cac-ac7c-d310725c558d
# ╠═7c69d12c-80a5-11ef-2674-e155a3483342
# ╠═effa7ed9-2ac4-4468-a474-e2bb662580fe
# ╠═1a6322d4-9deb-4709-aa2e-df7d8be5b16f
# ╠═11537f91-e16b-45f0-9768-6df842371d36
# ╠═268d3015-b8d3-48d9-b74a-062e258e0ec1
# ╟─62105832-df1f-4834-8da6-c542e22207d1
# ╠═dea6f143-7916-4765-92f6-2bfb97a72835
# ╟─41543c0c-d7c4-447b-a268-0d356c88d92c
# ╠═633ecdee-7e2f-4de8-833a-21cd0351c1f1
# ╟─87f1966e-3b07-4f9d-8fc4-7b7aa4319d50
# ╠═f379d43c-9300-41f4-b0fc-3c9d749e3105
# ╟─33e099bc-ac4b-4b5f-88e7-20f4463c98ef
# ╠═0e13f848-0efb-4775-9e3e-518b32588a79
# ╠═181c3fe6-d087-42e2-b175-3fb84c42e3e8
# ╟─dd669f14-989b-45ee-87d8-5d9cf282fafd
# ╠═1e9c3944-0cd4-40da-9014-a9153d4e95ed
# ╟─322cb530-65a5-4973-86f8-01ccc2439cc4
# ╠═c1caca5f-4cfd-4f22-82b4-7925002359e6
# ╠═3e5607fd-2a8a-4a1a-9e7b-3f23ef216fad
# ╠═c1b9c0c3-00f8-4199-b07f-8888f1be625c
# ╠═d6d04013-e0e4-49d5-a450-07ae164bfaa3
# ╠═0e680044-e4e1-4f39-a5c5-afa5c53fc7a7
# ╠═d7541b93-4c49-4dcd-bda0-91e447f44596
# ╟─e7faa647-79cd-4247-a3f2-9868c7b9d4ca
# ╠═3a412a98-4a2d-4bfb-8053-b0f480fae921
# ╟─b38bbed4-8721-4e92-a546-f7926cc07dd3
# ╠═420dce0e-4757-48d9-84ec-7ddfac2fdff6
# ╠═99902640-fee3-4502-9c7e-cb08834bad0b
# ╠═c79c6684-1b03-41b5-aa90-ef8c7a8eb69c
# ╟─ec774495-c0be-47a4-9d2c-b48159c07013
# ╠═d082e07c-3b42-4362-bebf-63356979a49b
# ╠═25688d24-5aee-43d3-aff9-b9efa0556070
# ╠═239a808c-0411-4542-ae68-6ae6af333bd2
# ╠═8e57bde1-5f97-483d-906e-8ebfb65016d0
# ╠═92c2ac3f-8034-4e9e-aadb-8bb166fbc948
# ╠═893253c3-f0b2-401f-b892-b23291bcf5c1
# ╠═6467dcaa-6bd6-45c7-8c08-b310a09b8b0b
# ╠═79e2f5d8-4609-4e9f-949e-6dc1f88c0b19
# ╠═d607e10e-854f-4652-9a34-9e22a188e315
# ╠═2795fd06-2f59-4e5b-829d-a8e428646790
# ╟─26976b6c-3954-4a41-a99b-c1aaebdc645d
# ╠═38d472ca-6347-4096-828d-fd1256130a59
# ╠═310710da-ebb2-4f54-b238-38d493a6a533
# ╠═e9964184-1a2a-4ab9-bc22-2705a25393ec
# ╠═f2d0ea9b-ede7-496a-8d1f-9f748f9d1666
# ╠═f5bc8b1a-3fee-48ed-9910-367a1cda632e
# ╠═9f3f780d-da6e-44b4-a002-c2f088681e9c
# ╠═6c3c7669-d594-425d-bf05-9aa217c7656e
# ╠═e9b3f541-d87d-4424-8b82-be1b7b3273d8