research/PSBoardDataBase
1
0
Fork 0
mirror of https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git synced 2025-06-07 21:45:43 +09:00
Code Issues Projects Releases Packages Wiki Activity Actions

add CSV download functions and set them as default and use in test

Browse source 
- replaced old ones from runtests
- need to remove CSVs in git repo
This commit is contained in:
Wataru Otsubo 2024-10-01 10:52:47 +09:00
parent 16d23ec4bb
commit d58a13793f
5 changed files with 107 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Project.toml
View file

@ -9,6 +9,7 @@ DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"

6
docs/src/about_software.md
View file

@ -48,3 +48,9 @@ Modules = [PSBoardDataBase]
```@autodocs
Modules = [QaqcMasterLog]
```
## `DownloadCSVs`
```@autodocs
Modules = [DownloadCSVs]
```

12
src/PSBoardDataBase.jl
View file

@ -10,7 +10,7 @@ using Dates
include("parse_qaqc_master_log.jl")
include("create_table.jl")
include("download_csv.jl")
include("import_data.jl")
"""
@ -37,11 +37,11 @@ Create database at `dbpath` and import data from CSV and master log files.
"""
function create_database_from_exported_csvs(
dbpath::AbstractString;
single_run_csv::AbstractString,
runlist_csv::AbstractString,
dispatch_csv::AbstractString,
hundred_csv::AbstractString,
jathubs_csv::AbstractString,
single_run_csv::AbstractString = DownloadCSVs.download_single_run_csv(),
runlist_csv::AbstractString = DownloadCSVs.download_runlist_csv(),
dispatch_csv::AbstractString = DownloadCSVs.download_dispatch_csv(),
hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
masterlog_dir::AbstractString,
)
db = create_database(dbpath)

61
src/download_csv.jl Normal file
View file

@ -0,0 +1,61 @@
"""
Functions to download result CSVs from Google Sheets.
All functions return the filename in `String`.
"""
module DownloadCSVs
using Downloads
"""
download_single_run_csv(outfile::AbstractString = tempname()) -> filename
# Example
```jldoctest
julia> file = PSBoardDataBase.DownloadCSVs.download_single_run_csv();
julia> using CSV
julia> using DataFrames
julia> CSV.read(file, DataFrame) isa DataFrame
true
```
"""
function download_single_run_csv(outfile::AbstractString = tempname())
URL_SINGLE_RUN_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=408695746"
Downloads.download(URL_SINGLE_RUN_CSV, outfile)
end
"""
download_runlist_csv(outfile::AbstractString = tempname()) -> filename
"""
function download_runlist_csv(outfile::AbstractString = tempname())
URL_RUNLIST_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=252134084"
Downloads.download(URL_RUNLIST_CSV, outfile)
end
"""
download_dispatch_csv(outfile::AbstractString = tempname()) -> filename
"""
function download_dispatch_csv(outfile::AbstractString = tempname())
URL_DISPATCH_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=707598141"
Downloads.download(URL_DISPATCH_CSV, outfile)
end
"""
download_hundred_run_csv(outfile::AbstractString = tempname()) -> filename
"""
function download_hundred_run_csv(outfile::AbstractString = tempname())
URL_HUNDRED_RUN_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=615256061"
Downloads.download(URL_HUNDRED_RUN_CSV, outfile)
end
"""
download_jathub_csv(outfile::AbstractString = tempname()) -> filename
"""
function download_jathub_csv(outfile::AbstractString = tempname())
URL_JATHUB_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=303843601"
Downloads.download(URL_JATHUB_CSV, outfile)
end
end # module DownloadCSVs

41
test/runtests.jl
View file

@ -16,17 +16,39 @@ true || include("../src/PSBoardDataBase.jl")
@test masterlog_metadata.shiftscript_version == v"1.0.2"
end
@testset "Download data csv" begin
out = tempname()
@test CSV.read(
PSBoardDataBase.DownloadCSVs.download_single_run_csv(out),
DataFrame,
) isa DataFrame
@test CSV.read(
PSBoardDataBase.DownloadCSVs.download_runlist_csv(out),
DataFrame,
) isa DataFrame
@test CSV.read(
PSBoardDataBase.DownloadCSVs.download_dispatch_csv(out),
DataFrame,
) isa DataFrame
@test CSV.read(
PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(out),
DataFrame,
) isa DataFrame
@test CSV.read(PSBoardDataBase.DownloadCSVs.download_jathub_csv(out), DataFrame) isa
DataFrame
end
@testset "prepare dataframe" begin
single_result_df =
CSV.read("input/PS board QAQC Data Base - 本番1回試験・追加検証試験.csv", DataFrame)
CSV.read(PSBoardDataBase.DownloadCSVs.download_single_run_csv(), DataFrame)
@test PSBoardDataBase.prepare_single_result_df(single_result_df) isa DataFrame
@test PSBoardDataBase.prepare_dispatch_table(
CSV.read("input/PS board QAQC Data Base - 出荷.csv", DataFrame),
CSV.read(PSBoardDataBase.DownloadCSVs.download_dispatch_csv(), DataFrame),
) isa DataFrame
@test PSBoardDataBase.prepare_100test_table(
CSV.read("input/PS board QAQC Data Base - 100回試験結果.csv", DataFrame),
CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame),
) isa DataFrame
end
@ -48,12 +70,14 @@ true || include("../src/PSBoardDataBase.jl")
end
@test PSBoardDataBase.insert_qaqc_campaign_id(db) |> isnothing
jathub_list_df = CSV.read("input/PS board QAQC Data Base - JATHub db.csv", DataFrame)
jathub_list_df =
CSV.read(PSBoardDataBase.DownloadCSVs.download_jathub_csv(), DataFrame)
@test PSBoardDataBase.insert_qaqc_positions(db, jathub_list_df) |> isnothing
single_result_df =
CSV.read("input/PS board QAQC Data Base - 本番1回試験・追加検証試験.csv", DataFrame)
runlist_table = CSV.read("input/PS board QAQC Data Base - RUNLIST.csv", DataFrame)
CSV.read(PSBoardDataBase.DownloadCSVs.download_single_run_csv(), DataFrame)
runlist_table =
CSV.read(PSBoardDataBase.DownloadCSVs.download_runlist_csv(), DataFrame)
@test PSBoardDataBase.add_psboard_ids(db, single_result_df) |> isnothing
@test PSBoardDataBase.add_qaqc_runlist_from_runlist(db, runlist_table) |> isnothing
@ -61,7 +85,8 @@ true || include("../src/PSBoardDataBase.jl")
@test PSBoardDataBase.add_qaqc_single_result(db, single_result_df, runlist_table) |>
isnothing
dispatch_table = CSV.read("input/PS board QAQC Data Base - 出荷.csv", DataFrame)
dispatch_table =
CSV.read(PSBoardDataBase.DownloadCSVs.download_dispatch_csv(), DataFrame)
@test PSBoardDataBase.add_qaqc_dispatch(db, dispatch_table) |> isnothing
@ -70,7 +95,7 @@ true || include("../src/PSBoardDataBase.jl")
isnothing
extra_100test_result_df =
CSV.read("input/PS board QAQC Data Base - 100回試験結果.csv", DataFrame)
CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame)
@test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df) |>
isnothing