add CSV download functions and set them as default and use in test

- replaced old ones from runtests
- need to remove CSVs in git repo

Project.toml

@@ -9,6 +9,7 @@ DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 

docs/src/about_software.md

@@ -48,3 +48,9 @@ Modules = [PSBoardDataBase]
 ```@autodocs
 Modules = [QaqcMasterLog]
 ```
+
+## `DownloadCSVs`
+
+```@autodocs
+Modules = [DownloadCSVs]
+```

src/PSBoardDataBase.jl

@@ -10,7 +10,7 @@ using Dates
 include("parse_qaqc_master_log.jl")
 
 include("create_table.jl")
-
+include("download_csv.jl")
 include("import_data.jl")
 
 """
@@ -37,11 +37,11 @@ Create database at `dbpath` and import data from CSV and master log files.
 """
 function create_database_from_exported_csvs(
     dbpath::AbstractString;
-    single_run_csv::AbstractString,
+    single_run_csv::AbstractString = DownloadCSVs.download_single_run_csv(),
-    runlist_csv::AbstractString,
+    runlist_csv::AbstractString = DownloadCSVs.download_runlist_csv(),
-    dispatch_csv::AbstractString,
+    dispatch_csv::AbstractString = DownloadCSVs.download_dispatch_csv(),
-    hundred_csv::AbstractString,
+    hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
-    jathubs_csv::AbstractString,
+    jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
     masterlog_dir::AbstractString,
 )
     db = create_database(dbpath)

src/download_csv.jl

@@ -0,0 +1,61 @@
+"""
+Functions to download result CSVs from Google Sheets.
+All functions return the filename in `String`.
+"""
+module DownloadCSVs
+
+using Downloads
+
+"""
+    download_single_run_csv(outfile::AbstractString = tempname()) -> filename
+
+# Example
+```jldoctest
+julia> file = PSBoardDataBase.DownloadCSVs.download_single_run_csv();
+
+julia> using CSV
+
+julia> using DataFrames
+
+julia> CSV.read(file, DataFrame) isa DataFrame
+true
+```
+"""
+function download_single_run_csv(outfile::AbstractString = tempname())
+    URL_SINGLE_RUN_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=408695746"
+    Downloads.download(URL_SINGLE_RUN_CSV, outfile)
+end
+
+"""
+    download_runlist_csv(outfile::AbstractString = tempname()) -> filename
+"""
+function download_runlist_csv(outfile::AbstractString = tempname())
+    URL_RUNLIST_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=252134084"
+    Downloads.download(URL_RUNLIST_CSV, outfile)
+end
+
+"""
+    download_dispatch_csv(outfile::AbstractString = tempname()) -> filename
+"""
+function download_dispatch_csv(outfile::AbstractString = tempname())
+    URL_DISPATCH_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=707598141"
+    Downloads.download(URL_DISPATCH_CSV, outfile)
+end
+
+"""
+    download_hundred_run_csv(outfile::AbstractString = tempname()) -> filename
+"""
+function download_hundred_run_csv(outfile::AbstractString = tempname())
+    URL_HUNDRED_RUN_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=615256061"
+    Downloads.download(URL_HUNDRED_RUN_CSV, outfile)
+end
+
+"""
+    download_jathub_csv(outfile::AbstractString = tempname()) -> filename
+"""
+function download_jathub_csv(outfile::AbstractString = tempname())
+    URL_JATHUB_CSV::String = "https://docs.google.com/spreadsheets/u/1/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=303843601"
+    Downloads.download(URL_JATHUB_CSV, outfile)
+end
+
+end # module DownloadCSVs

test/runtests.jl

@@ -16,17 +16,39 @@ true || include("../src/PSBoardDataBase.jl")
         @test masterlog_metadata.shiftscript_version == v"1.0.2"
     end
 
+    @testset "Download data csv" begin
+        out = tempname()
+        @test CSV.read(
+            PSBoardDataBase.DownloadCSVs.download_single_run_csv(out),
+            DataFrame,
+        ) isa DataFrame
+        @test CSV.read(
+            PSBoardDataBase.DownloadCSVs.download_runlist_csv(out),
+            DataFrame,
+        ) isa DataFrame
+        @test CSV.read(
+            PSBoardDataBase.DownloadCSVs.download_dispatch_csv(out),
+            DataFrame,
+        ) isa DataFrame
+        @test CSV.read(
+            PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(out),
+            DataFrame,
+        ) isa DataFrame
+        @test CSV.read(PSBoardDataBase.DownloadCSVs.download_jathub_csv(out), DataFrame) isa
+              DataFrame
+    end
+
     @testset "prepare dataframe" begin
         single_result_df =
-            CSV.read("input/PS board QAQC Data Base - 本番1回試験・追加検証試験.csv", DataFrame)
+            CSV.read(PSBoardDataBase.DownloadCSVs.download_single_run_csv(), DataFrame)
         @test PSBoardDataBase.prepare_single_result_df(single_result_df) isa DataFrame
 
         @test PSBoardDataBase.prepare_dispatch_table(
-            CSV.read("input/PS board QAQC Data Base - 出荷.csv", DataFrame),
+            CSV.read(PSBoardDataBase.DownloadCSVs.download_dispatch_csv(), DataFrame),
         ) isa DataFrame
 
         @test PSBoardDataBase.prepare_100test_table(
-            CSV.read("input/PS board QAQC Data Base - 100回試験結果.csv", DataFrame),
+            CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame),
         ) isa DataFrame
     end
 
@@ -48,12 +70,14 @@ true || include("../src/PSBoardDataBase.jl")
         end
 
         @test PSBoardDataBase.insert_qaqc_campaign_id(db) |> isnothing
-        jathub_list_df = CSV.read("input/PS board QAQC Data Base - JATHub db.csv", DataFrame)
+        jathub_list_df =
+            CSV.read(PSBoardDataBase.DownloadCSVs.download_jathub_csv(), DataFrame)
         @test PSBoardDataBase.insert_qaqc_positions(db, jathub_list_df) |> isnothing
 
         single_result_df =
-            CSV.read("input/PS board QAQC Data Base - 本番1回試験・追加検証試験.csv", DataFrame)
+            CSV.read(PSBoardDataBase.DownloadCSVs.download_single_run_csv(), DataFrame)
-        runlist_table = CSV.read("input/PS board QAQC Data Base - RUNLIST.csv", DataFrame)
+        runlist_table =
+            CSV.read(PSBoardDataBase.DownloadCSVs.download_runlist_csv(), DataFrame)
 
         @test PSBoardDataBase.add_psboard_ids(db, single_result_df) |> isnothing
         @test PSBoardDataBase.add_qaqc_runlist_from_runlist(db, runlist_table) |> isnothing
@@ -61,7 +85,8 @@ true || include("../src/PSBoardDataBase.jl")
         @test PSBoardDataBase.add_qaqc_single_result(db, single_result_df, runlist_table) |>
               isnothing
 
-        dispatch_table = CSV.read("input/PS board QAQC Data Base - 出荷.csv", DataFrame)
+        dispatch_table =
+            CSV.read(PSBoardDataBase.DownloadCSVs.download_dispatch_csv(), DataFrame)
 
         @test PSBoardDataBase.add_qaqc_dispatch(db, dispatch_table) |> isnothing
 
@@ -70,7 +95,7 @@ true || include("../src/PSBoardDataBase.jl")
                   isnothing
 
             extra_100test_result_df =
-                CSV.read("input/PS board QAQC Data Base - 100回試験結果.csv", DataFrame)
+                CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame)
 
             @test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df) |>
                   isnothing