"""
    insert_version_info(db::SQLite.DB)

Insert version information of this software as string.
"""
function insert_version_info(db::SQLite.DB)
    stmt_insert_version = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO versions VALUES (:converter)
        """,
    )
    @info "converter version info" pkgversion(@__MODULE__) |> string
    DBInterface.execute(
        stmt_insert_version,
        (; converter = pkgversion(@__MODULE__) |> string),
    )

    nothing
end

"""
    insert_qaqc_campaign_id(db::SQLite.DB)

Fill qaqc_campaigns table in `db`.
"""
function insert_qaqc_campaign_id(db::SQLite.DB)
    campaigns = [1, 2, 3, 4, 5]
    dates = [
        (DateTime(2024, 7, 22), DateTime(2024, 7, 24)),
        (DateTime(2024, 8, 6), DateTime(2024, 8, 9)),
        (DateTime(2024, 9, 10), DateTime(2024, 9, 12)),
        (DateTime(2024, 9, 30), DateTime(2024, 10, 4)),
        (DateTime(2024, 11, 11), DateTime(2024, 11, 14)),
    ]
    stmt_insert_campaigns = DBInterface.prepare(
        db,
        sql"INSERT INTO qaqc_campaigns VALUES (:id, :start_date, :end_date, :note)",
    )
    DBInterface.executemany(
        stmt_insert_campaigns,
        (
            id = campaigns,
            start_date = dates .|> (x -> x[1]) .|> string,
            end_date = dates .|> (x -> x[2]) .|> string,
            note = fill(nothing, size(campaigns)),
        ),
    )

    nothing
end

"""
    insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame)

Fill qaqc_positions table in `db`.
Argument `jathub_db_table` is for skew for each positions.
"""
function insert_qaqc_positions(db::SQLite.DB, jathub_db_table::DataFrame)
    dropmissing!(jathub_db_table, :psb_position)
    transform!(
        jathub_db_table,
        Symbol("立ち上がり [ns]") => ByRow(Float64) => Symbol("立ち上がり [ns]"),
    )

    stmt_insert_positions = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO qaqc_positions
        VALUES(
            :id,
            :name,
            :station,
            :position,
            :rising_ns
        )
        """,
    )
    DBInterface.executemany(
        stmt_insert_positions,
        (
            id = 1:18,
            name = ["B-$i-$j" for i in 0:1 for j in 1:9],
            station = [fill(0, 9); fill(1, 9)],
            position = [collect(1:9); collect(1:9)],
            rising_ns = [
                filter(
                    :psb_position => (s -> !ismissing(s) && s == "B-$i-$j"),
                    jathub_db_table,
                ).var"立ち上がり [ns]" |> first for i in 0:1 for j in 1:9
            ],
        ),
    )

    nothing
end

"""
    prepare_single_result_df(single_result_table::DataFrame)

Common preprocess(format) function for single result table.

# Detail
- convert `timestamp` to [`DateTime`](@extref Dates.DateTime)
"""
function prepare_single_result_df(single_result_table::DataFrame)
    df = copy(single_result_table, copycols = true)

    # timestamp format: 2024-08-07T06:18:09Z
    # ignore the last 'Z' => [1:end-1]
    transform!(
        df,
        :timestamp =>
            ByRow(s -> ismissing(s) ? missing : DateTime(s[1:(end - 1)])) => :timestamp,
    )

    return df
end

"""
    prepare_runlist_df(runlist_table::DataFrame)

Common preprocess(format) function for runlist table.
"""
function prepare_runlist_df(runlist_table::DataFrame)
    df = copy(runlist_table, copycols = true)
end

"""
    add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame)

Add PS Board IDs from single test result table.
Assume that all PS Boards are included in `single_result_table`.
"""
function add_psboard_ids(db::SQLite.DB, single_result_table::DataFrame)
    df = combine(groupby(single_result_table, :motherboard_id)) do df
        if df.daughterboard |> unique |> length |> ==(1)
            return (daughterboard = df.daughterboard |> first,)
        end
        df = sort(df, :timestamp)
        dropmissing!(df, :daughterboard)
        return (daughterboard = df.daughterboard |> last,)
    end
    filter!(:motherboard_id => !=(999999), df)

    stmt_insert_psbid = DBInterface.prepare(
        db,
        sql"INSERT INTO ps_boards VALUES (:psbid, :daughterboardid)",
    )
    DBInterface.executemany(
        stmt_insert_psbid,
        (psbid = df.motherboard_id, daughterboardid = df.daughterboard),
    )

    nothing
end

"""
    add_qaqc_runlist_from_runlist(db::SQLite.DB, runlist_table::DataFrame)

Add QAQC runs to `qaqc_runs` table in `db` from RUNLIST csv.
"""
function add_qaqc_runlist_from_runlist(db::SQLite.DB, runlist_table::DataFrame)
    stmt_insert_runid = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO qaqc_runs
        VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
        """,
    )
    runlist_table = dropmissing(runlist_table, Symbol("Run ID"))
    @assert allunique(runlist_table, Symbol("Run ID"))
    for row in eachrow(runlist_table)
        try
            DBInterface.execute(
                stmt_insert_runid,
                (
                    runid = row.var"Run ID",
                    campaign_id = row.var"Campaign ID",
                    run_datetime = nothing,
                    note = row.comment,
                    shifter = row.var"Shifter name",
                    logfile = nothing,
                    shiftscript_ver = nothing,
                ),
            )
        catch e
            @error "error in putting run list" e
            @info "row" row
        end
    end
    nothing
end

"""
    get_campaign_id_from_run_id(runid::Integer)
"""
function get_campaign_id_from_run_id(runid::Integer)
    if runid < 63
        1
    elseif runid < 98
        2
    elseif runid < 188
        3
    elseif runid < 293
        4
    elseif runid < Inf # TODO: update this at the end of 5th campaign
        5
    else
        @error "Fix this function"
        DomainError("runid $(runid) is not registered to the software")
    end
end

"""
    add_qaqc_single_result(
        db::SQLite.DB,
        single_result_table::DataFrame,
        runlist_table::DataFrame,
    ) -> nothing

Fill `qaqc_single_run_results` in `db` from single result table DataFrame.
Additionally, it
1. automatically add `runid` if it's not in `qaqc_runs` table in `db`.
2. automatically update fields in `qaqc_runs` table.
"""
function add_qaqc_single_result(
    db::SQLite.DB,
    single_result_table::DataFrame,
    runlist_table::DataFrame,
)
    single_result_table = prepare_single_result_df(single_result_table)

    position_id_map =
        ["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
            (i, s) = x
            s => i
        end) |> Dict

    stmt_search_runid = DBInterface.prepare(
        db,
        sql"""
        SELECT id
        FROM qaqc_runs
        WHERE id = :runid
        """,
    )
    stmt_insert_runid = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO qaqc_runs
        VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
        """,
    )
    stmt_update_runid = DBInterface.prepare(
        db,
        sql"""
        UPDATE qaqc_runs
        SET run_datetime = :run_datetime, shifter = :shifter, logfile = :logfile, shiftscript_ver = :shiftscript_ver
        WHERE id = :runid
        """,
    )
    stmt_insert_result = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO 
            qaqc_single_run_results(
                runid,
                psboard_id,
                daughterboard_id,
                position,
                resistance_test_passed,
                qspip,
                recov,
                power,
                clock,
                asdtp,
                reset,
                qaqc_result,
                note
            )
        VALUES (
                :runid,
                :psboard_id,
                :daughterboard_id,
                :position,
                :resistance_test_passed,
                :qspip,
                :recov,
                :power,
                :clock,
                :asdtp,
                :reset,
                :qaqc_result,
                :note
            )
        """,
    )
    stmt_insert_resistance = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO qaqc_resistance_check(psb_id, passed)
        VALUES (:psboard_id, :ispassed)
        """,
    )

    for row in eachrow(single_result_table)
        if ismissing(row.runid)
            @assert contains("resistance")(row.comment) || contains("CN15")(row.comment) "Unexpected row with id $(row.motherboard_id) $(row.comment)"

            DBInterface.execute(
                stmt_insert_resistance,
                (psboard_id = row.motherboard_id, ispassed = false),
            )

            continue
        end

        # Add run if it's not in `qaqc_runs` table
        # or update info on the run (such as datetime)
        if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
            campaign_id = get_campaign_id_from_run_id(row.runid)
            comment = let
                row_run = filter(
                    Symbol("Run ID") => x -> !ismissing(x) && x == row.runid,
                    runlist_table,
                )
                if !isempty(row_run)
                    row_run.comment
                else
                    ""
                end
            end

            DBInterface.execute(
                stmt_insert_runid,
                (
                    runid = row.runid,
                    campaign_id = campaign_id,
                    run_datetime = row.timestamp |> string,
                    note = comment,
                    shifter = row.shifter,
                    logfile = row.qaqc_log_file,
                    shiftscript_ver = row.shiftscript_ver,
                ),
            )
        else
            DBInterface.execute(
                stmt_update_runid,
                (
                    runid = row.runid,
                    run_datetime = row.timestamp |> string,
                    shifter = row.shifter,
                    logfile = row.qaqc_log_file,
                    shiftscript_ver = row.shiftscript_ver,
                ),
            )
        end

        # resistance
        DBInterface.execute(
            stmt_insert_resistance,
            (psboard_id = row.motherboard_id, ispassed = true),
        )

        # main result
        DBInterface.execute(
            stmt_insert_result,
            (
                runid = row.runid,
                psboard_id = row.motherboard_id,
                daughterboard_id = row.daughterboard,
                position = position_id_map[row.position],
                resistance_test_passed = true,
                qspip = row.qspip,
                recov = row.recov,
                power = row.power,
                clock = row.clock,
                asdtp = row.asdtp,
                reset = row.reset,
                qaqc_result = row.qaqc_result,
                note = row.comment,
            ),
        )
    end

    nothing
end

"""
    prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame

Format `qaqc_dispatch` DataFrame from exported CSV.
Used in [`add_qaqc_dispatch`](@ref).
"""
function prepare_dispatch_table(raw_dispatch_table::DataFrame)::DataFrame
    df = copy(raw_dispatch_table, copycols = true)
    transform!(
        df,
        [Symbol("Column$i") for i in 2:ncol(df)] => ByRow((s...) -> join(s |> skipmissing)) => :comment,
    )
    select!(df, [1, ncol(df)])
    rename!(df, [:psboard_id, :comment])

    transform!(
        df,
        :psboard_id => (vs -> accumulate(vs; init = 0) do x, y
            ispsbid = startswith("PS")
            if ispsbid(y)
                x
            else
                match(r"(\d+)", y) |> first
            end
        end) => :campaign_id,
    )
    transform!(
        df,
        :psboard_id =>
            ByRow(s -> startswith("PS")(s) ? parse(Int64, s[3:end]) : missing) =>
                :psboard_id,
    )
    dropmissing!(df, :psboard_id)

    df
end

"""
    add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)

Fill `qaqc_dispatch` table in `db` from `dispatch_table`.
"""
function add_qaqc_dispatch(db::SQLite.DB, dispatch_table::DataFrame)
    dispatch_table = prepare_dispatch_table(dispatch_table)

    # TODO: provide datetime
    stmt_insert_dispatch = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO qaqc_dispatch(qaqc_campaign_id, psb_id, source_place, destination, time)
        VALUES (:campaign_id, :psboard_id, "KEK", "GND", NULL)
        """,
    )

    DBInterface.executemany(
        stmt_insert_dispatch,
        (campaign_id = dispatch_table.campaign_id, psboard_id = dispatch_table.psboard_id),
    )

    nothing
end

"""
    add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString) -> nothing

Add qaqc run list from master log files in `logs_dir`.
Currently, it adds long runs and run with id 20-23 only (since normal runs are added from single run results table).
"""
function add_qaqc_runlist_from_masterlogs(db::SQLite.DB, logs_dir::AbstractString)
    stmt_search_runid = DBInterface.prepare(
        db,
        sql"""
        SELECT id, run_datetime
        FROM qaqc_runs
        WHERE id = :runid
        """,
    )
    stmt_update_runid = DBInterface.prepare(
        db,
        sql"""
        UPDATE qaqc_runs
        SET run_datetime = :run_datetime, shifter = :shifter, logfile = :logfile, shiftscript_ver = :shiftscript_ver
        WHERE id = :runid
        """,
    )
    stmt_insert_runid = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO qaqc_runs
        VALUES (:runid, :campaign_id, :run_datetime, :note, :shifter, :logfile, :shiftscript_ver)
        """,
    )

    is_run_to_add(log_file) = begin
        m = match(r"(\d+)\.log", log_file)
        contains("_long.log")(log_file) ||
        !isnothing(m) && (
            begin
                num = parse(Int64, m[1])
                20 <= num <= 23 || num == 27 || num == 28
            end
        )
    end

    longrun_logs = readdir(logs_dir; join = true) |> filter(is_run_to_add)
    # longrun_logs = readdir(logs_dir; join = true) |> filter(contains("_long.log"))
    for longrun_log in longrun_logs
        run_metadata = QaqcMasterLog.parse_master_log(longrun_log)
        if isnothing(run_metadata)
            continue
        end
        current_rundb =
            DBInterface.execute(stmt_search_runid, (; runid = run_metadata.runid)) |>
            DataFrame
        if !isempty(current_rundb.id)
            # runid is already in the database
            if any(ismissing, current_rundb.run_datetime)
                # add timestamp, logfile, ...etc
                DBInterface.execute(
                    stmt_update_runid,
                    (
                        runid = run_metadata.runid,
                        run_datetime = run_metadata.timestamp |> string,
                        shifter = run_metadata.shifters,
                        logfile = splitdir(longrun_log) |> last,
                        shiftscript_ver = run_metadata.shiftscript_version,
                    ),
                )
            end
            continue
        end

        DBInterface.execute(
            stmt_insert_runid,
            (
                runid = run_metadata.runid,
                campaign_id = get_campaign_id_from_run_id(run_metadata.runid),
                run_datetime = run_metadata.timestamp |> string,
                note = "",
                shifter = run_metadata.shifters,
                logfile = splitdir(longrun_log) |> last,
                shiftscript_ver = run_metadata.shiftscript_version,
            ),
        )
    end

    nothing
end

"""
    prepare_100test_table(table::DataFrame)::DataFrame

Format 100test result `table` from exported CSV.
Used in [`add_qaqc_100test_result`](@ref).

# Detail
- Format `motherboard ID`s
    - `PS00xxxx` -> `Int64(xxxx)`
    - `xxxx` -> `Int64(xxxx)`
- For `psbid == 484` and `runid == 115` results, make all result fields to `missing` since they contain abnormal strings.
"""
function prepare_100test_table(table::DataFrame)::DataFrame
    df = copy(table, copycols = true)

    transform!(
        df,
        Symbol("motherboard ID") =>
            ByRow(s -> if startswith("PS")(s)
                parse(Int64, s[3:end])
            else
                parse(Int64, s)
            end) => :motherboard_id,
    )
    transform!(
        df,
        Cols(:motherboard_id, :runid, 4:13) =>
            ByRow((psbid, runid, items...) -> if psbid == 484 && runid == 115
                missings(items |> length)
            else
                items
            end) => names(df)[4:13],
    )

    df
end

"""
    get_num_tests_for_extra_runs(runid::Int64)

Get number of tests for extra QAQC runs.
They are usually 100.

Current abnormal runs:

| runid | # of runs |
|-------|-----------|
|     99|        246|
"""
function get_num_tests_for_extra_runs(runid::Int64)
    if runid == 99
        246
    else
        100
    end
end

"""
    add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) -> nothing

Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
which is converted from a raw exported CSV.

# Detail
- skips psboards in `resistance_test_passed` with `passed == false`
"""
function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
    position_id_map =
        ["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
            (i, s) = x
            s => i
        end) |> Dict

    table = prepare_100test_table(table)

    stmt_search_runid = DBInterface.prepare(
        db,
        sql"""
        SELECT id
        FROM qaqc_runs
        WHERE id = :runid
        """,
    )
    stmt_search_resistance_error = DBInterface.prepare(
        db,
        sql"""
        SELECT psb_id
        FROM qaqc_resistance_check
        WHERE psb_id = :psboard_id AND passed = 0
        """,
    )
    stmt_insert_result = DBInterface.prepare(
        db,
        sql"""
        INSERT INTO
            qaqc_extra_run_results (
                runid,
                psboard_id,
                position,
                num_tests,
                insufficient_reset_with_10,
                reset_failed_though_reconfig_done,
                always_hit_flag_true,
                dac_is_0,
                bcid_shift,
                efficiency_99percent,
                bcid_fail_111,
                bcid_fail_000,
                low_efficiency,
                bcid_fail,
                invalid_register_value,
                power_out_of_range,
                note
            )
        VALUES (
            :runid,
            :psboard_id,
            :position,
            :num_tests,
            :insufficient_reset_with_10,
            :reset_failed_though_reconfig_done,
            :always_hit_flag_true,
            :dac_is_0,
            :bcid_shift,
            :efficiency_99percent,
            :bcid_fail_111,
            :bcid_fail_000,
            :low_efficiency,
            :bcid_fail,
            :invalid_register_value,
            :power_out_of_range,
            :note
        )
        """,
    )

    for row in eachrow(table)
        if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
            # search for resistance error
            if !isempty(
                DBInterface.execute(
                    stmt_search_resistance_error,
                    (; psboard_id = row.motherboard_id),
                ),
            )
                continue
            end
            error("Runid $(row.runid) not found in `qaqc_runs` table.")
        end

        DBInterface.execute(
            stmt_insert_result,
            (
                runid = row.runid,
                psboard_id = row.motherboard_id,
                position = position_id_map[row.position],
                num_tests = get_num_tests_for_extra_runs(row.runid),
                insufficient_reset_with_10 = row.var"10回reset足りず",
                reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない",
                always_hit_flag_true = row.var"always_hit_flag",
                dac_is_0 = row.var"DAC = 0",
                bcid_shift = row.var"DAC = 0",
                efficiency_99percent = row.var"efficiency 99%",
                bcid_fail_111 = row.var"BCID 0:0:0",
                bcid_fail_000 = row.var"BCID 1:1:1",
                low_efficiency = row.var"low efficiency",
                bcid_fail = row.var"BCID fail",
                invalid_register_value = row.var"invalid register values",
                power_out_of_range = row.var"power out of range",
                note = row.Column20,
            ),
        )
    end

    nothing
end

"""
    add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)

Insert skew measurement result from slave logs with name `psbid_runid_clk.txt`.
See [`ClockParser.get_skew`](@ref) for parse detail.

# Abnormal logs:
- `48_nagoya_irradition_...`: skipped
- `630_190`: broken file
"""
function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
    stmt_insert_skew_to_single_result = DBInterface.prepare(
        db,
        sql"""
        UPDATE qaqc_single_run_results
        SET lvds_tx_skew = :skew
        WHERE runid = :runid AND psboard_id = :psbid
        """,
    )
    clk_files =
        readdir(joinpath(logs_dir, "main"), join = true) |>
        filter(endswith("_clk.txt")) |>
        filter(!contains("nagoya"))

    DBInterface.transaction(db) do
        for file in clk_files
            m = match(r"^(?<psbid>\d+)_(?<runid>\d+)_clk.txt$", splitdir(file) |> last)
            if isnothing(m)
                error("Invalid filename $(file)")
            end

            if m[:psbid] == "630" && m[:runid] == "190"
                @debug "skipping... (psbid=630 runid=190 is broken)"
                continue
            end

            DBInterface.execute(
                stmt_insert_skew_to_single_result,
                (skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]),
            )
        end
    end

    nothing
end

"""
    add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)

Extract QAQC results from slave log files for single runs.
Slave log files are expected to located in certain format under `logs_dir`.
"""
function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
    exclude_runs =
        ((runid = 51, reason = "clock only"), (runid = 175, reason = "broken files"))

    stmt_insert_slave_result_to_single_result = DBInterface.prepare(
        db,
        sql"""
        UPDATE qaqc_single_run_results
        SET
            power_3v3d = :power_3v3d,
            power_3v3a = :power_3v3a,
            power_n3va = :power_n3va
        WHERE
            runid = :runid AND psboard_id = :psbid
        """,
    )
    runids =
        DBInterface.execute(
            db,
            sql"""
            SELECT id
            FROM qaqc_runs
            """,
        ) |> Tables.columntable |> (tbl -> tbl.id)

    slave_files =
        readdir(joinpath(logs_dir, "main"), join = true) |>
        filter(contains(r"\d+_\d+\.txt"))

    DBInterface.transaction(db) do
        for file in slave_files
            psbid, runid, islongrun =
                SlaveLogParser.get_psbid_runid_from_filename(basename(file))
            @assert !islongrun

            # exclusion

            exclude_cond = Iterators.filter(exclude_runs) do cond
                runid == cond.runid
            end
            if !isempty(exclude_cond)
                @debug "skipping runid = $(runid) for $(first(exclude_cond).reason)"
                continue
            end

            if !(runid in runids)
                slave_result = SlaveLogParser.parse_slavelog_file(file)
                @debug "runid: $(runid) not in run list (psbid: $(psbid))"
                continue
            end

            # main

            slave_result = SlaveLogParser.parse_slavelog_file(file)

            @assert length(slave_result.power) == 1 "Too many power results for single run"

            DBInterface.execute(
                stmt_insert_slave_result_to_single_result,
                (;
                    power_3v3d = slave_result.power[1].result_3v3d,
                    power_3v3a = slave_result.power[1].result_3v3a,
                    power_n3va = slave_result.power[1].result_n3va,
                    runid,
                    psbid,
                ),
            )
        end
    end

    nothing
end