From eff62d7f09ac49932e40cb571716ad46eec7c6d6 Mon Sep 17 00:00:00 2001 From: Wataru Otsubo Date: Sun, 14 Jul 2024 11:12:01 +0900 Subject: [PATCH] new: CSV output --- Cargo.lock | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 5 +- README.md | 8 ++ src/main.rs | 149 ++++++++++++++++++++++++++------- tests/cli.rs | 10 +-- 5 files changed, 368 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76786f9..2d5d999 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,6 +117,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.6.0" @@ -254,6 +260,72 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "difflib" version = "0.4.0" @@ -289,6 +361,12 @@ dependencies = [ "log", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.9" @@ -314,6 +392,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "globset" version = "0.4.14" @@ -338,12 +422,30 @@ dependencies = [ "walkdir", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "humantime" version = "2.1.0" @@ -373,6 +475,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "ignore" version = "0.4.22" @@ -389,12 +497,40 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + [[package]] name = "js-sys" version = "0.3.69" @@ -434,6 +570,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.19" @@ -449,6 +591,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "predicates" version = "3.1.0" @@ -499,11 +647,14 @@ dependencies = [ "clap", "clap-verbosity-flag", "clap_derive", + "csv", "env_logger", "log", "predicates", "regex", "semver", + "serde", + "serde_with", ] [[package]] @@ -557,6 +708,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + [[package]] name = "same-file" version = "1.0.6" @@ -571,6 +728,9 @@ name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -592,6 +752,47 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e73139bc5ec2d45e6c5fd85be5a46949c1c39a4c18e56915f5eb4c12f975e377" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b80d3d6b56b64335c0180e5ffde23b3c5e08c14c585b51a15bd0e95393f46703" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "strsim" version = "0.11.1" @@ -627,6 +828,37 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "unicode-ident" version = "1.0.12" diff --git a/Cargo.toml b/Cargo.toml index bfad1c2..9a67c4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,8 +11,11 @@ env_logger = "0.11.3" clap-verbosity-flag = "2.2" anyhow = "1.0" chrono = { version = "0.4", features = ["serde"] } -semver = "1.0" +semver = { version = "1.0", features = ["serde"] } regex = "1.10" +serde = { version = "1.0", features = ["derive"] } +serde_with = "3.8" +csv = "1.3" [dev-dependencies] assert_cmd = "2.0" diff --git a/README.md b/README.md index 1ace4bc..aa84e5f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # psb_qaqc +WIP + ## For developers ### build / run ```sh @@ -18,6 +20,12 @@ cargo doc --open ``` ## TODO +- add more unittests - test for various cases +- split into muds(maybe as lib crates) +- organize existing feature as a subcommand +- add csv check/validate subcommand - skew measurement? - rename bin/project + +check `TODO`s in comments. diff --git a/src/main.rs b/src/main.rs index 27e3cbb..2d70f5f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,9 @@ use core::{panic, str}; use std::{ collections::HashMap, - env, fmt::Display, fs::File, - io::{BufRead, BufReader, Write}, + io::{BufRead, BufReader}, path::{self, PathBuf}, str::FromStr, }; @@ -15,17 +14,8 @@ use clap::Parser; use log::{debug, error, info, trace, warn}; use regex::Regex; use semver::Version; - -fn get_default_output_path() -> PathBuf { - PathBuf::from_str("outpu.csv").expect("Error") -} -fn get_default_log_path() -> PathBuf { - let mut path = env::current_exe().unwrap(); - path.pop(); - path.push("log"); - path.push("debug.log"); - path -} +use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; /// Parse master jathub logfile for PS Board QAQC and write out to CSV. #[derive(Parser, Debug)] @@ -145,7 +135,7 @@ impl FromStr for PsbId { /// Temporary stores raw values. /// TODO: specify each filed types.(maybe multi values) #[derive(Debug)] -struct BoardResult { +struct MasterBoardResult { id: PsbId, qspif: u8, qspip: u8, @@ -158,11 +148,12 @@ struct BoardResult { /// Full result for a single QAQC run from a log file on JATHub master. #[derive(Debug)] -struct MasterLogResult { +pub struct MasterLogResult { version: Version, datetime: DateTime, shifter: String, - board_results: HashMap, + board_results: HashMap, + filename: String, } /// Get version of shift script. @@ -193,7 +184,7 @@ fn extract_position_id(line: &str) -> Result<(Position, PsbId)> { impl MasterLogResult { /// Parse log file on master jathub. - fn parse_file(file: impl BufRead) -> Result { + fn parse_file(file: impl BufRead, filename: String) -> Result { let mut lines = file.lines(); let version = { @@ -322,7 +313,7 @@ impl MasterLogResult { .context(format!("No board on pos {}", pos))? .clone(); - let result = BoardResult { + let result = MasterBoardResult { id: psbid, qspif: parts .get(1) @@ -370,17 +361,70 @@ impl MasterLogResult { datetime, shifter, board_results, + filename, }) } - - fn write(&self, f: impl Write) -> Result<()> { - todo!() - } } -fn check_csv_header(f: impl BufRead) -> Result<()> { - // f.lines() - todo!() +/// All information on QAQC stored in the database. +/// +/// Everything without `Option` is available from master log. +/// +/// TODO: use pos? shifter? version? +#[serde_as] +#[derive(PartialEq, Debug, Serialize, Deserialize)] +pub struct PsbQaqcResult { + motherboard_id: u32, + daughterboard_id: Option, + #[serde_as(as = "DisplayFromStr")] + position: Position, + qspif: u8, + qspip: u8, + recov: u8, + clock: u8, + regac: u8, + asdtp: u8, + qaqc_result: u32, + lvds_tx_skew: Option, + ppl_lock_reset_count: Option, + timestamp: DateTime, + qaqc_log_file: String, + firmware_ver: Option, + parameter_ver: Option, + fpga_dna: Option, + comment: String, +} + +impl PsbQaqcResult { + /// Expand [`MasterLogResult`] to [`PsbQaqcResult`]. + /// Filling unavailable fileds with [`None`]s. + pub fn from_masterlogresult(result: MasterLogResult) -> Vec { + let mut converted = vec![]; + for (pos, boardresult) in result.board_results { + let new = PsbQaqcResult { + motherboard_id: boardresult.id.id, + daughterboard_id: None, + position: pos, + qspif: boardresult.qspif, + qspip: boardresult.qspip, + recov: boardresult.recov, + clock: boardresult.clock, + regac: boardresult.regac, + asdtp: boardresult.asdtp, + qaqc_result: boardresult.done.into(), + lvds_tx_skew: None, + ppl_lock_reset_count: None, + timestamp: result.datetime, + qaqc_log_file: result.filename.clone(), + firmware_ver: None, + parameter_ver: None, + fpga_dna: None, + comment: "".to_string(), + }; + converted.push(new); + } + converted + } } fn main() -> Result<()> { @@ -397,9 +441,17 @@ fn main() -> Result<()> { ); let result = { - let file = File::open(args.master_log)?; + let file = File::open(args.master_log.clone())?; let reader = BufReader::new(file); - MasterLogResult::parse_file(reader)? + MasterLogResult::parse_file( + reader, + args.master_log + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(), + )? }; debug!("{:?}", result); @@ -407,8 +459,36 @@ fn main() -> Result<()> { // let file = File::options() // .read(true) // .open(args.outfile)?; - // } // + // let mut rdr = csv::Reader::from_reader(file); + // rdr.records(); + // } + + let expanded_results = PsbQaqcResult::from_masterlogresult(result); + + let mut wtr = match args.outfile.exists() { + true => { + let file = File::options().read(true).append(true).open(args.outfile)?; + csv::WriterBuilder::new() + .has_headers(false) + .from_writer(file) + } + false => { + println!("Creating new file: {}", args.outfile.display()); + let file = File::options() + .create_new(true) + .write(true) + .open(args.outfile)?; + csv::WriterBuilder::new() + .has_headers(true) + .from_writer(file) + } + }; + for result in expanded_results { + wtr.serialize(result)?; + } + wtr.flush()?; + Ok(()) } @@ -417,8 +497,11 @@ mod test { use std::str::FromStr; use chrono::{DateTime, Utc}; + use semver::Version; - use crate::{extract_position_id, extract_shifter_line, Position, PositionLayer, PsbId}; + use crate::{ + extract_position_id, extract_shifter_line, extract_version, Position, PositionLayer, PsbId, + }; #[test] fn parse_position() { @@ -454,6 +537,14 @@ mod test { assert!(DateTime::parse_from_str("Date: 2024-06-20T08:42:01+0000", "Date: %+").is_ok()); } + #[test] + fn parse_version() { + assert_eq!( + extract_version("Shift script: 1.0.0").unwrap(), + Version::new(1, 0, 0) + ) + } + #[test] fn parse_shifter_line() { assert_eq!(extract_shifter_line("Shifters: foo").unwrap(), "foo"); diff --git a/tests/cli.rs b/tests/cli.rs index 0384cf1..32bccbf 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -15,9 +15,8 @@ mod integrated_test { .arg("./example_logs/valid/20240620_083537.log") .arg(test_out.as_path()) .assert() - .success(); - // .failure() - // .stderr(predicate::str::contains("not yet implemented")); + .success() + .stdout(predicate::str::contains("Creating new file")); Ok(()) } @@ -32,9 +31,8 @@ mod integrated_test { .arg("./example_logs/valid/20240620_093537.log") .arg(test_out.as_path()) .assert() - .success(); - // .failure() - // .stderr(predicate::str::contains("not yet implemented")); + .success() + .stdout(predicate::str::contains("Creating new file")); Ok(()) }