new: CSV output

Cargo.lock

@@ -117,6 +117,12 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
 
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
 [[package]]
 name = "bitflags"
 version = "2.6.0"
@@ -254,6 +260,72 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
+[[package]]
+name = "csv"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "darling"
+version = "0.20.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.20.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.20.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "deranged"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
+dependencies = [
+ "powerfmt",
+ "serde",
+]
+
 [[package]]
 name = "difflib"
 version = "0.4.0"
@@ -289,6 +361,12 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "errno"
 version = "0.3.9"
@@ -314,6 +392,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "globset"
 version = "0.4.14"
@@ -338,12 +422,30 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hashbrown"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+
 [[package]]
 name = "heck"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -373,6 +475,12 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
 [[package]]
 name = "ignore"
 version = "0.4.22"
@@ -389,12 +497,40 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+ "serde",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.5",
+ "serde",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
 
+[[package]]
+name = "itoa"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+
 [[package]]
 name = "js-sys"
 version = "0.3.69"
@@ -434,6 +570,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
 
+[[package]]
+name = "num-conv"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -449,6 +591,12 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "predicates"
 version = "3.1.0"
@@ -499,11 +647,14 @@ dependencies = [
  "clap",
  "clap-verbosity-flag",
  "clap_derive",
+ "csv",
  "env_logger",
  "log",
  "predicates",
  "regex",
  "semver",
+ "serde",
+ "serde_with",
 ]
 
 [[package]]
@@ -557,6 +708,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "ryu"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -571,6 +728,9 @@ name = "semver"
 version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "serde"
@@ -592,6 +752,47 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.120"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_with"
+version = "3.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e73139bc5ec2d45e6c5fd85be5a46949c1c39a4c18e56915f5eb4c12f975e377"
+dependencies = [
+ "base64",
+ "chrono",
+ "hex",
+ "indexmap 1.9.3",
+ "indexmap 2.2.6",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "serde_with_macros",
+ "time",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "3.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b80d3d6b56b64335c0180e5ffde23b3c5e08c14c585b51a15bd0e95393f46703"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -627,6 +828,37 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
+[[package]]
+name = "time"
+version = "0.3.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
+dependencies = [
+ "deranged",
+ "itoa",
+ "num-conv",
+ "powerfmt",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+
+[[package]]
+name = "time-macros"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.12"

Cargo.toml

@@ -11,8 +11,11 @@ env_logger = "0.11.3"
 clap-verbosity-flag = "2.2"
 anyhow = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
-semver = "1.0"
+semver = { version = "1.0", features = ["serde"] }
 regex = "1.10"
+serde = { version = "1.0", features = ["derive"] }
+serde_with = "3.8"
+csv = "1.3"
 
 [dev-dependencies]
 assert_cmd = "2.0"

README.md

@@ -1,5 +1,7 @@
 # psb_qaqc
 
+WIP
+
 ## For developers
 ### build / run
 ```sh
@@ -18,6 +20,12 @@ cargo doc --open
 ```
 
 ## TODO
+- add more unittests
 - test for various cases
+- split into muds(maybe as lib crates)
+- organize existing feature as a subcommand
+- add csv check/validate subcommand
 - skew measurement?
 - rename bin/project
+
+check `TODO`s in comments.

src/main.rs

@@ -1,10 +1,9 @@
 use core::{panic, str};
 use std::{
     collections::HashMap,
-    env,
     fmt::Display,
     fs::File,
-    io::{BufRead, BufReader, Write},
+    io::{BufRead, BufReader},
     path::{self, PathBuf},
     str::FromStr,
 };
@@ -15,17 +14,8 @@ use clap::Parser;
 use log::{debug, error, info, trace, warn};
 use regex::Regex;
 use semver::Version;
-
+use serde::{Deserialize, Serialize};
-fn get_default_output_path() -> PathBuf {
+use serde_with::{serde_as, DisplayFromStr};
-    PathBuf::from_str("outpu.csv").expect("Error")
-}
-fn get_default_log_path() -> PathBuf {
-    let mut path = env::current_exe().unwrap();
-    path.pop();
-    path.push("log");
-    path.push("debug.log");
-    path
-}
 
 /// Parse master jathub logfile for PS Board QAQC and write out to CSV.
 #[derive(Parser, Debug)]
@@ -145,7 +135,7 @@ impl FromStr for PsbId {
 /// Temporary stores raw values.
 /// TODO: specify each filed types.(maybe multi values)
 #[derive(Debug)]
-struct BoardResult {
+struct MasterBoardResult {
     id: PsbId,
     qspif: u8,
     qspip: u8,
@@ -158,11 +148,12 @@ struct BoardResult {
 
 /// Full result for a single QAQC run from a log file on JATHub master.
 #[derive(Debug)]
-struct MasterLogResult {
+pub struct MasterLogResult {
     version: Version,
     datetime: DateTime<Utc>,
     shifter: String,
-    board_results: HashMap<Position, BoardResult>,
+    board_results: HashMap<Position, MasterBoardResult>,
+    filename: String,
 }
 
 /// Get version of shift script.
@@ -193,7 +184,7 @@ fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
 
 impl MasterLogResult {
     /// Parse log file on master jathub.
-    fn parse_file(file: impl BufRead) -> Result<MasterLogResult> {
+    fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
         let mut lines = file.lines();
 
         let version = {
@@ -322,7 +313,7 @@ impl MasterLogResult {
                     .context(format!("No board on pos {}", pos))?
                     .clone();
 
-                let result = BoardResult {
+                let result = MasterBoardResult {
                     id: psbid,
                     qspif: parts
                         .get(1)
@@ -370,17 +361,70 @@ impl MasterLogResult {
             datetime,
             shifter,
             board_results,
+            filename,
         })
     }
-
-    fn write(&self, f: impl Write) -> Result<()> {
-        todo!()
-    }
 }
 
-fn check_csv_header(f: impl BufRead) -> Result<()> {
+/// All information on QAQC stored in the database.
-    // f.lines()
+///
-    todo!()
+/// Everything without `Option` is available from master log.
+///
+/// TODO: use pos? shifter? version?
+#[serde_as]
+#[derive(PartialEq, Debug, Serialize, Deserialize)]
+pub struct PsbQaqcResult {
+    motherboard_id: u32,
+    daughterboard_id: Option<u32>,
+    #[serde_as(as = "DisplayFromStr")]
+    position: Position,
+    qspif: u8,
+    qspip: u8,
+    recov: u8,
+    clock: u8,
+    regac: u8,
+    asdtp: u8,
+    qaqc_result: u32,
+    lvds_tx_skew: Option<u32>,
+    ppl_lock_reset_count: Option<i32>,
+    timestamp: DateTime<Utc>,
+    qaqc_log_file: String,
+    firmware_ver: Option<Version>,
+    parameter_ver: Option<Version>,
+    fpga_dna: Option<u64>,
+    comment: String,
+}
+
+impl PsbQaqcResult {
+    /// Expand [`MasterLogResult`] to [`PsbQaqcResult`].
+    /// Filling unavailable fileds with [`None`]s.
+    pub fn from_masterlogresult(result: MasterLogResult) -> Vec<Self> {
+        let mut converted = vec![];
+        for (pos, boardresult) in result.board_results {
+            let new = PsbQaqcResult {
+                motherboard_id: boardresult.id.id,
+                daughterboard_id: None,
+                position: pos,
+                qspif: boardresult.qspif,
+                qspip: boardresult.qspip,
+                recov: boardresult.recov,
+                clock: boardresult.clock,
+                regac: boardresult.regac,
+                asdtp: boardresult.asdtp,
+                qaqc_result: boardresult.done.into(),
+                lvds_tx_skew: None,
+                ppl_lock_reset_count: None,
+                timestamp: result.datetime,
+                qaqc_log_file: result.filename.clone(),
+                firmware_ver: None,
+                parameter_ver: None,
+                fpga_dna: None,
+                comment: "".to_string(),
+            };
+            converted.push(new);
+        }
+        converted
+    }
 }
 
 fn main() -> Result<()> {
@@ -397,9 +441,17 @@ fn main() -> Result<()> {
     );
 
     let result = {
-        let file = File::open(args.master_log)?;
+        let file = File::open(args.master_log.clone())?;
         let reader = BufReader::new(file);
-        MasterLogResult::parse_file(reader)?
+        MasterLogResult::parse_file(
+            reader,
+            args.master_log
+                .file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .to_string(),
+        )?
     };
     debug!("{:?}", result);
 
@@ -407,8 +459,36 @@ fn main() -> Result<()> {
     //     let file = File::options()
     //         .read(true)
     //         .open(args.outfile)?;
-    // }
     //
+    //     let mut rdr = csv::Reader::from_reader(file);
+    //     rdr.records();
+    // }
+
+    let expanded_results = PsbQaqcResult::from_masterlogresult(result);
+
+    let mut wtr = match args.outfile.exists() {
+        true => {
+            let file = File::options().read(true).append(true).open(args.outfile)?;
+            csv::WriterBuilder::new()
+                .has_headers(false)
+                .from_writer(file)
+        }
+        false => {
+            println!("Creating new file: {}", args.outfile.display());
+            let file = File::options()
+                .create_new(true)
+                .write(true)
+                .open(args.outfile)?;
+            csv::WriterBuilder::new()
+                .has_headers(true)
+                .from_writer(file)
+        }
+    };
+    for result in expanded_results {
+        wtr.serialize(result)?;
+    }
+    wtr.flush()?;
+
     Ok(())
 }
 
@@ -417,8 +497,11 @@ mod test {
     use std::str::FromStr;
 
     use chrono::{DateTime, Utc};
+    use semver::Version;
 
-    use crate::{extract_position_id, extract_shifter_line, Position, PositionLayer, PsbId};
+    use crate::{
+        extract_position_id, extract_shifter_line, extract_version, Position, PositionLayer, PsbId,
+    };
 
     #[test]
     fn parse_position() {
@@ -454,6 +537,14 @@ mod test {
         assert!(DateTime::parse_from_str("Date: 2024-06-20T08:42:01+0000", "Date: %+").is_ok());
     }
 
+    #[test]
+    fn parse_version() {
+        assert_eq!(
+            extract_version("Shift script: 1.0.0").unwrap(),
+            Version::new(1, 0, 0)
+        )
+    }
+
     #[test]
     fn parse_shifter_line() {
         assert_eq!(extract_shifter_line("Shifters: foo").unwrap(), "foo");

tests/cli.rs

@@ -15,9 +15,8 @@ mod integrated_test {
             .arg("./example_logs/valid/20240620_083537.log")
             .arg(test_out.as_path())
             .assert()
-            .success();
+            .success()
-        // .failure()
+            .stdout(predicate::str::contains("Creating new file"));
-        // .stderr(predicate::str::contains("not yet implemented"));
 
         Ok(())
     }
@@ -32,9 +31,8 @@ mod integrated_test {
             .arg("./example_logs/valid/20240620_093537.log")
             .arg(test_out.as_path())
             .assert()
-            .success();
+            .success()
-        // .failure()
+            .stdout(predicate::str::contains("Creating new file"));
-        // .stderr(predicate::str::contains("not yet implemented"));
 
         Ok(())
     }