From b4760aed587079f6c61b0a9212976a8ee6a37897 Mon Sep 17 00:00:00 2001 From: Wataru Otsubo Date: Thu, 15 May 2025 18:30:27 +0900 Subject: [PATCH] new: 8th campaign CSV download and preprocessing --- Manifest.toml | 189 +++++++++++++++++++++++++++++++++++---- Project.toml | 2 + src/DownloadCSVs.jl | 4 + src/PSBoardDataBase.jl | 11 ++- src/import_data.jl | 20 +++++ src/sql/create_table.sql | 8 ++ 6 files changed, 216 insertions(+), 18 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 5a18a2c..9765d8f 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.11.5" manifest_format = "2.0" -project_hash = "47f0621871082a27225026b076304121300bf44d" +project_hash = "a9fa03ac044d2e070b6689aa0efcf1ed6a422332" [[deps.ANSIColoredPrinters]] git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" @@ -14,6 +14,35 @@ git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" version = "0.4.5" +[[deps.Accessors]] +deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"] +git-tree-sha1 = "3b86719127f50670efe356bc11073d84b4ed7a5d" +uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" +version = "0.1.42" + + [deps.Accessors.extensions] + AxisKeysExt = "AxisKeys" + IntervalSetsExt = "IntervalSets" + LinearAlgebraExt = "LinearAlgebra" + StaticArraysExt = "StaticArrays" + StructArraysExt = "StructArrays" + TestExt = "Test" + UnitfulExt = "Unitful" + + [deps.Accessors.weakdeps] + AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.ArgCheck]] +git-tree-sha1 = "f9e9a66c9b7be1ad7372bbd9b062d9230c30c5ce" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.5.0" + [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" version = "1.1.2" @@ -27,10 +56,37 @@ git-tree-sha1 = "4ec6b48702dacc5994a835c1189831755e4e76ef" uuid = "15f4f7f2-30c1-5605-9d31-71845cf9641f" version = "2.2.0" +[[deps.BangBang]] +deps = ["Accessors", "ConstructionBase", "InitialValues", "LinearAlgebra"] +git-tree-sha1 = "26f41e1df02c330c4fa1e98d4aa2168fdafc9b1f" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.4.4" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTablesExt = "Tables" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + [[deps.CSV]] deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" @@ -58,6 +114,30 @@ deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" version = "1.1.1+0" +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" +weakdeps = ["InverseFunctions"] + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + +[[deps.ConstructionBase]] +git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.8" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" @@ -95,16 +175,26 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + [[deps.DocStringExtensions]] git-tree-sha1 = "e7b7e6f178525d17c720ab9c081e4ef04429f860" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.4" [[deps.Documenter]] -deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "9d733459cea04dcf1c41522ec25c31576387be8a" +deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] +git-tree-sha1 = "22ca291bada1e8af75dcb5b41f9e1d2da91c2986" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.10.1" +version = "1.11.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] @@ -150,10 +240,10 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll"] -git-tree-sha1 = "04eff47b1354d702c3a85e8ab23d539bb7d5957e" +deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.3.1" +version = "1.4.0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] @@ -167,6 +257,11 @@ git-tree-sha1 = "b6d6bfdd7ce25b0f9b2f6b3dd56b2673a66c8770" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" version = "0.2.5" +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + [[deps.InlineStrings]] git-tree-sha1 = "6a9fde685a7ac1eb3495f8e812c5a7c3711c2d5e" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" @@ -185,6 +280,16 @@ deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" version = "1.11.0" +[[deps.InverseFunctions]] +git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb" +uuid = "3587e190-3f89-42d0-90ee-14403ec27112" +version = "0.1.17" +weakdeps = ["Dates", "Test"] + + [deps.InverseFunctions.extensions] + InverseFunctionsDatesExt = "Dates" + InverseFunctionsTestExt = "Test" + [[deps.InvertedIndices]] git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe" uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" @@ -196,10 +301,16 @@ uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" [[deps.JLD2]] -deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] -git-tree-sha1 = "1059c071429b4753c0c869b75c859c44ba09a526" +deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "TranscodingStreams"] +git-tree-sha1 = "8e071648610caa2d3a5351aba03a936a0c37ec61" uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.5.12" +version = "0.5.13" + + [deps.JLD2.extensions] + UnPackExt = "UnPack" + + [deps.JLD2.weakdeps] + UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] @@ -268,9 +379,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" [[deps.MacroTools]] -git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472" +git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.15" +version = "0.5.16" [[deps.Markdown]] deps = ["Base64"] @@ -288,6 +399,12 @@ deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" version = "2.28.6+0" +[[deps.MicroCollections]] +deps = ["Accessors", "BangBang", "InitialValues"] +git-tree-sha1 = "44d32db644e84c75dab479f1bc15ee76a1a3618f" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.2.0" + [[deps.Missings]] deps = ["DataAPI"] git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" @@ -311,11 +428,17 @@ deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" version = "0.3.27+1" +[[deps.OpenSSH_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] +git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" +version = "10.0.1+0" + [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "a9697f1d06cc3eb3fb3ad49cc67f2cfabaac31ea" +git-tree-sha1 = "9216a80ff3682833ac4b733caa8c00390620ba5d" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.16+0" +version = "3.5.0+0" [[deps.OrderedCollections]] git-tree-sha1 = "cc4054e898b852042d7b503313f7ad03de99c3dd" @@ -329,9 +452,9 @@ version = "10.42.0+1" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "44f6c1f38f77cafef9450ff93946c53bd9ca16ff" +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.2" +version = "2.8.3" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] @@ -424,6 +547,12 @@ version = "1.4.8" uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" version = "1.11.0" +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "c5391c6ace3bc430ca630251d02ea9687169ca68" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.2" + [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" @@ -434,6 +563,12 @@ git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" version = "1.2.1" +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] git-tree-sha1 = "0feb6b9031bd5c51f9072393eb5ab3efd31bf9e4" @@ -507,6 +642,28 @@ git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.11.3" +[[deps.Transducers]] +deps = ["Accessors", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "ConstructionBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "SplittablesBase", "Tables"] +git-tree-sha1 = "7deeab4ff96b85c5f72c824cae53a1398da3d1cb" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.84" + + [deps.Transducers.extensions] + TransducersAdaptExt = "Adapt" + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" diff --git a/Project.toml b/Project.toml index 2259492..bff9b60 100644 --- a/Project.toml +++ b/Project.toml @@ -16,6 +16,7 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +Transducers = "28d57a85-8fef-5791-bfe6-a80928e7c999" [weakdeps] InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -38,6 +39,7 @@ SQLite = "1" StaticArrays = "1.9" Tables = "1" Test = "1.10" +Transducers = "0.4.84" julia = "1.10" [extras] diff --git a/src/DownloadCSVs.jl b/src/DownloadCSVs.jl index ddc0fc9..4954b69 100644 --- a/src/DownloadCSVs.jl +++ b/src/DownloadCSVs.jl @@ -58,4 +58,8 @@ function download_jathub_csv(outfile::AbstractString = tempname()) Downloads.download(URL_JATHUB_CSV, outfile) end +function download_second_run_csv(outfile::AbstractString = tempname()) + URL_SECOND_CSV::String = "https://docs.google.com/spreadsheets/u/0/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/export?format=csv&id=128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU&gid=717277847" + Downloads.download(URL_SECOND_CSV, outfile) +end end # module DownloadCSVs diff --git a/src/PSBoardDataBase.jl b/src/PSBoardDataBase.jl index 119316b..b44167a 100644 --- a/src/PSBoardDataBase.jl +++ b/src/PSBoardDataBase.jl @@ -22,7 +22,10 @@ using .DispatchChecker export create_database -@deprecate create_database_from_exported_csvs(dbpath::AbstractString; kw...) create_database(dbpath; kw...) false +@deprecate create_database_from_exported_csvs(dbpath::AbstractString; kw...) create_database( + dbpath; + kw..., +) false """ create_database( @@ -78,13 +81,15 @@ function create_database( runlist_table, dispatch_table, extra_100test_result_df, - jathubs_table = + jathubs_table, + eighth_campaign_df = map(( (:single_run_csv, DownloadCSVs.download_single_run_csv), (:runlist_csv, DownloadCSVs.download_runlist_csv), (:dispatch_csv, DownloadCSVs.download_dispatch_csv), (:hundred_csv, DownloadCSVs.download_hundred_run_csv), (:jathubs_csv, DownloadCSVs.download_jathub_csv), + (:eighth_campaign_df, DownloadCSVs.download_second_run_csv), )) do (key, dlfunc) if !haskey(kw, key) Channel{DataFrame}(1) do ch @@ -122,6 +127,8 @@ function create_database( add_skew_from_slave_clk_logs(db, slavelog_dir) add_slavelog_result(db, slavelog_dir, jld2_slavelog) + add_eighth_campaign_result(db, eighth_campaign_df) + if !isnothing(jld2_slavelog) close(jld2_slavelog) end diff --git a/src/import_data.jl b/src/import_data.jl index ab557bc..cb5b17a 100644 --- a/src/import_data.jl +++ b/src/import_data.jl @@ -976,3 +976,23 @@ function add_slavelog_result( nothing end + +function prepare_eighth_campaign_result(result_df) + # counting + popat!(result_df, 1) + popat!(result_df, 1) + popat!(result_df, 1) + + transform!( + result_df, + :motherboard_id => ByRow(passmissing(id -> parse(Int64, id))) => :motherboard_id, + ) + + falied_boards = [895, 896, 1142, 1276, 1094, 1095] + + result_df +end + +function add_eighth_campaign_result(db, result_df) + nothing +end diff --git a/src/sql/create_table.sql b/src/sql/create_table.sql index f0a2ecc..e2abe0d 100644 --- a/src/sql/create_table.sql +++ b/src/sql/create_table.sql @@ -176,3 +176,11 @@ AS qaqc_runs.id ASC, qaqc_positions.id; +CREATE TABLE qaqc_eighth_run_results( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + runid INTEGER NOT NULL, + psboard_id INTEGER NOT NULL, + position INTEGER NOT NULL, + qspip INTEGER NOT NULL, + note TEXT +);