research/PSBoardDataBase
1
0
Fork 0
mirror of https://gitlab.cern.ch/wotsubo/PSBoardDataBase.git synced 2025-06-08 05:55:42 +09:00
Code Issues Projects Releases Packages Wiki Activity Actions

update: parallelize CSV downloads

Browse source
This commit is contained in:
Wataru Otsubo 2024-11-11 20:50:43 +09:00
parent 99a0d9a1de
commit 1868270dab
1 changed files with 26 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/PSBoardDataBase.jl
View file

@ -48,20 +48,36 @@ Create database at `dbpath` and import data from CSV and master log files.
""" """
function create_database_from_exported_csvs( function create_database_from_exported_csvs(
dbpath::AbstractString; dbpath::AbstractString;
single_run_csv::AbstractString = DownloadCSVs.download_single_run_csv(),
runlist_csv::AbstractString = DownloadCSVs.download_runlist_csv(),
dispatch_csv::AbstractString = DownloadCSVs.download_dispatch_csv(),
hundred_csv::AbstractString = DownloadCSVs.download_hundred_run_csv(),
jathubs_csv::AbstractString = DownloadCSVs.download_jathub_csv(),
masterlog_dir::AbstractString, masterlog_dir::AbstractString,
slavelog_dir::AbstractString, slavelog_dir::AbstractString,
kw...
) )
db = create_database(dbpath) db = create_database(dbpath)
single_result_df = CSV.read(single_run_csv, DataFrame)
runlist_table = CSV.read(runlist_csv, DataFrame) single_result_df,
dispatch_table = CSV.read(dispatch_csv, DataFrame) runlist_table,
extra_100test_result_df = CSV.read(hundred_csv, DataFrame) dispatch_table,
jathubs_table = CSV.read(jathubs_csv, DataFrame) extra_100test_result_df,
jathubs_table =
map((
(:single_run_csv, DownloadCSVs.download_single_run_csv),
(:runlist_csv, DownloadCSVs.download_runlist_csv),
(:dispatch_csv, DownloadCSVs.download_dispatch_csv),
(:hundred_csv, DownloadCSVs.download_hundred_run_csv),
(:jathubs_csv, DownloadCSVs.download_jathub_csv),
)) do (key, dlfunc)
if !haskey(kw, key)
Channel{DataFrame}(1) do ch
csvfile = dlfunc()
df = CSV.read(csvfile, DataFrame)
put!(ch, df)
end
else
Channel{DataFrame}(1) do ch
put!(ch, CSV.read(kw[key], DataFrame))
end
end
end .|> first
insert_version_info(db) insert_version_info(db)
insert_qaqc_campaign_id(db) insert_qaqc_campaign_id(db)