### A Pluto.jl notebook ###
# v0.19.46

using Markdown
using InteractiveUtils

# ╔═╡ 7c69d12c-80a5-11ef-2674-e155a3483342
begin
    using Pkg
    Pkg.activate("..")
    Pkg.status()
    true || include("../src/PSBoardDataBase.jl")
end

# ╔═╡ effa7ed9-2ac4-4468-a474-e2bb662580fe
begin
    using PSBoardDataBase
    using SQLite
    using DataFrames
    using DBInterface
    using Tables
    using CairoMakie
    using Statistics
    using PlutoUI
end

# ╔═╡ f25e7e08-8a73-4cac-ac7c-d310725c558d
md"""
# Detailed research on clock skew measurement
- distribution of clock skews
- consistency of multiple measurement on the same board
"""

# ╔═╡ 1a6322d4-9deb-4709-aa2e-df7d8be5b16f
TableOfContents()

# ╔═╡ 11537f91-e16b-45f0-9768-6df842371d36
db = SQLite.DB("../psboard_qaqc.db")

# ╔═╡ 268d3015-b8d3-48d9-b74a-062e258e0ec1
SQLite.tables(db)

# ╔═╡ 62105832-df1f-4834-8da6-c542e22207d1
md"""
## Single runs
"""

# ╔═╡ dea6f143-7916-4765-92f6-2bfb97a72835
qaqc_single_results =
    DBInterface.execute(db, sql"select * from qaqc_single_run_results") |> DataFrame

# ╔═╡ 41543c0c-d7c4-447b-a268-0d356c88d92c
md"""
## PS Board list
"""

# ╔═╡ 633ecdee-7e2f-4de8-833a-21cd0351c1f1
DBInterface.execute(
    db,
    sql"""
    select * from ps_boards
    """,
) |> DataFrame

# ╔═╡ 87f1966e-3b07-4f9d-8fc4-7b7aa4319d50
md"""
## Run list
"""

# ╔═╡ f379d43c-9300-41f4-b0fc-3c9d749e3105
qaqc_runs = DBInterface.execute(db, sql"select * from qaqc_runs") |> DataFrame

# ╔═╡ 33e099bc-ac4b-4b5f-88e7-20f4463c98ef
md"""
## Positions
"""

# ╔═╡ 0e13f848-0efb-4775-9e3e-518b32588a79
qaqc_positions = DBInterface.execute(db, sql"select * from qaqc_positions") |> DataFrame

# ╔═╡ 181c3fe6-d087-42e2-b175-3fb84c42e3e8
position_id_skew_map = select(qaqc_positions, [:id, :rising_ns]) |> Tables.rowtable |> Dict

# ╔═╡ 322cb530-65a5-4973-86f8-01ccc2439cc4
md"""
# Clock Analysis
main part
"""

# ╔═╡ c1caca5f-4cfd-4f22-82b4-7925002359e6
clk_files =
    readdir("../test/input/slavelogs/main/", join = true) |>
    filter(endswith("_clk.txt")) |>
    filter(!contains("nagoya")) |>
    filter(!contains("630_190"))

# ╔═╡ 3e5607fd-2a8a-4a1a-9e7b-3f23ef216fad
"""
Get `(psbid, runid)`.
"""
function parse_filename(filename::AbstractString)
    m = match(r"(?<psbid>\d+)_(?<runid>\d+)_clk\.txt", filename)
    parse(Int64, m[:psbid]), parse(Int64, m[:runid])
end

# ╔═╡ c1b9c0c3-00f8-4199-b07f-8888f1be625c
parse_filename("190_23_clk.txt")

# ╔═╡ d6d04013-e0e4-49d5-a450-07ae164bfaa3
# Get skew and rise up time from clock measurement files
# Use measurements recorded in qaqc_single_results
df_rawskews =
    clk_files .|>
    (
        file -> begin
            skew_width = PSBoardDataBase.ClockParser.get_skew_and_riseup(file)
            psbid, runid = parse_filename(file)
            (psbid = psbid, runid = runid, skew = skew_width[1], width = skew_width[2])
        end
    ) |>
    filter(
        x ->
            filter(
                [:psboard_id, :runid] =>
                    ((psbid, runid) -> (psbid == x.psbid && runid == x.runid)),
                qaqc_single_results,
            ) |> !isempty,
    ) |>
    DataFrame

# ╔═╡ d7541b93-4c49-4dcd-bda0-91e447f44596
# substract result of measurements of position dependency
df_skews = let
    df = leftjoin(
        df_rawskews,
        @view(qaqc_single_results[!, [:psboard_id, :runid, :position]]),
        on = [:psbid => :psboard_id, :runid],
    )
    leftjoin!(df, @view(qaqc_positions[!, [:id, :rising_ns]]), on = [:position => :id])
    transform!(df, [:skew, :rising_ns] => ByRow((x, y) -> x - y) => :skew)
    select!(df, Not(:rising_ns))
    select!(df, Not(:position))
end

# ╔═╡ 420dce0e-4757-48d9-84ec-7ddfac2fdff6
stephist(df_skews.width |> skipmissing |> collect)

# ╔═╡ d082e07c-3b42-4362-bebf-63356979a49b
gdf_skews_on_psbid = groupby(df_skews, :psbid)

# ╔═╡ 25688d24-5aee-43d3-aff9-b9efa0556070
combine(nrow, gdf_skews_on_psbid)

# ╔═╡ 239a808c-0411-4542-ae68-6ae6af333bd2
df_nrow_ordered = let
    df = combine(nrow, gdf_skews_on_psbid)
    sort!(df, :nrow, rev = true)
end

# ╔═╡ 8e57bde1-5f97-483d-906e-8ebfb65016d0
@view(df_nrow_ordered[findall(>(1), df_nrow_ordered.nrow), :])

# ╔═╡ 92c2ac3f-8034-4e9e-aadb-8bb166fbc948
df_skew_stats = let
    df = combine(
        gdf_skews_on_psbid,
        sdf -> begin
            if nrow(sdf) == 1
                (; mean_skew = mean(sdf.skew), std_skew = missing, n = 1)
            else
                (; mean_skew = mean(sdf.skew), std_skew = std(sdf.skew), n = nrow(sdf))
            end
        end,
    )
    dropmissing!(df)
    df
end

# ╔═╡ 893253c3-f0b2-401f-b892-b23291bcf5c1
fig_skew_stats = let
    fig, ax, sc = scatter(
        df_skew_stats.mean_skew,
        df_skew_stats.std_skew,
        marker = :x,
        color = (Makie.wong_colors()[1], 0.8),
        axis = (title = "skew mean vs std", xlabel = "mean", ylabel = "std"),
    )
    text!(
        ax,
        df_skew_stats.mean_skew,
        df_skew_stats.std_skew,
        text = string.(df_skew_stats.psbid),
        color = (:gray, 0.5),
    )
    fig
end

# ╔═╡ 6467dcaa-6bd6-45c7-8c08-b310a09b8b0b
save("clock_skew_stats.svg", fig_skew_stats)

# ╔═╡ 79e2f5d8-4609-4e9f-949e-6dc1f88c0b19
df_skew_stats_abnormals =
    filter([:mean_skew, :std_skew] => ((m, s) -> m > -5 && s > 1), df_skew_stats)

# ╔═╡ d607e10e-854f-4652-9a34-9e22a188e315
let
    df = df_skew_stats_abnormals
    fig, ax, sc = scatter(
        df.mean_skew,
        df.std_skew,
        marker = :x,
        color = (Makie.wong_colors()[1], 0.8),
        axis = (title = "skew mean vs std", xlabel = "mean", ylabel = "std"),
    )
    text!(
        ax,
        df.mean_skew,
        df.std_skew,
        text = string.(df.psbid) .* "," .* string.(df.n),
        color = (:gray, 0.7),
    )
    fig
end

# ╔═╡ 2795fd06-2f59-4e5b-829d-a8e428646790
md"""
## 分散が異常に大きいやつ
基本的に予想通り、分散は小さく複数回の測定で整合的な結果が得られているが、いくつか例外があった。

はじめはpsbid 127(4回測定)が含まれていたが、これはデータベースの編集ミスであることがわかり、修正した結果、消えた。

### psbid: 291
- run: 83, 94
- 83でcommunication error(SFP半抜け)
### psbid: 460
- run: 105, 132
- psbid 444と同じく電源の抜き差しによってクロックの0と1000が繰り返されたパターン
- 追試に送られてる
### psbid: 545 (**問題の**)
- run: 126, 132
- どちらも測定結果自体には変なところはない
  - どちらも1回だけ立ち上がりがある
  - 立ち上がりもそれほど長くない
- 126が電源が不安定なときだったかもしれないが、記録がない
"""

# ╔═╡ Cell order:
# ╟─f25e7e08-8a73-4cac-ac7c-d310725c558d
# ╠═7c69d12c-80a5-11ef-2674-e155a3483342
# ╠═effa7ed9-2ac4-4468-a474-e2bb662580fe
# ╠═1a6322d4-9deb-4709-aa2e-df7d8be5b16f
# ╠═11537f91-e16b-45f0-9768-6df842371d36
# ╠═268d3015-b8d3-48d9-b74a-062e258e0ec1
# ╟─62105832-df1f-4834-8da6-c542e22207d1
# ╠═dea6f143-7916-4765-92f6-2bfb97a72835
# ╟─41543c0c-d7c4-447b-a268-0d356c88d92c
# ╠═633ecdee-7e2f-4de8-833a-21cd0351c1f1
# ╟─87f1966e-3b07-4f9d-8fc4-7b7aa4319d50
# ╠═f379d43c-9300-41f4-b0fc-3c9d749e3105
# ╟─33e099bc-ac4b-4b5f-88e7-20f4463c98ef
# ╠═0e13f848-0efb-4775-9e3e-518b32588a79
# ╠═181c3fe6-d087-42e2-b175-3fb84c42e3e8
# ╟─322cb530-65a5-4973-86f8-01ccc2439cc4
# ╠═c1caca5f-4cfd-4f22-82b4-7925002359e6
# ╠═3e5607fd-2a8a-4a1a-9e7b-3f23ef216fad
# ╠═c1b9c0c3-00f8-4199-b07f-8888f1be625c
# ╠═d6d04013-e0e4-49d5-a450-07ae164bfaa3
# ╠═d7541b93-4c49-4dcd-bda0-91e447f44596
# ╠═420dce0e-4757-48d9-84ec-7ddfac2fdff6
# ╠═d082e07c-3b42-4362-bebf-63356979a49b
# ╠═25688d24-5aee-43d3-aff9-b9efa0556070
# ╠═239a808c-0411-4542-ae68-6ae6af333bd2
# ╠═8e57bde1-5f97-483d-906e-8ebfb65016d0
# ╠═92c2ac3f-8034-4e9e-aadb-8bb166fbc948
# ╠═893253c3-f0b2-401f-b892-b23291bcf5c1
# ╠═6467dcaa-6bd6-45c7-8c08-b310a09b8b0b
# ╠═79e2f5d8-4609-4e9f-949e-6dc1f88c0b19
# ╠═d607e10e-854f-4652-9a34-9e22a188e315
# ╠═2795fd06-2f59-4e5b-829d-a8e428646790