new: CSV output

This commit is contained in:
Wataru Otsubo 2024-07-14 11:12:01 +09:00
parent a14eb4ea48
commit eff62d7f09
5 changed files with 368 additions and 36 deletions

232
Cargo.lock generated
View file

@ -117,6 +117,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
version = "2.6.0"
@ -254,6 +260,72 @@ version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "csv"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
dependencies = [
"csv-core",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "csv-core"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
dependencies = [
"memchr",
]
[[package]]
name = "darling"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]]
name = "deranged"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
dependencies = [
"powerfmt",
"serde",
]
[[package]]
name = "difflib"
version = "0.4.0"
@ -289,6 +361,12 @@ dependencies = [
"log",
]
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.9"
@ -314,6 +392,12 @@ dependencies = [
"num-traits",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "globset"
version = "0.4.14"
@ -338,12 +422,30 @@ dependencies = [
"walkdir",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "humantime"
version = "2.1.0"
@ -373,6 +475,12 @@ dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "ignore"
version = "0.4.22"
@ -389,12 +497,40 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
"serde",
]
[[package]]
name = "indexmap"
version = "2.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
dependencies = [
"equivalent",
"hashbrown 0.14.5",
"serde",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
[[package]]
name = "itoa"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "js-sys"
version = "0.3.69"
@ -434,6 +570,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
[[package]]
name = "num-conv"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-traits"
version = "0.2.19"
@ -449,6 +591,12 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "predicates"
version = "3.1.0"
@ -499,11 +647,14 @@ dependencies = [
"clap",
"clap-verbosity-flag",
"clap_derive",
"csv",
"env_logger",
"log",
"predicates",
"regex",
"semver",
"serde",
"serde_with",
]
[[package]]
@ -557,6 +708,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "ryu"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
version = "1.0.6"
@ -571,6 +728,9 @@ name = "semver"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
dependencies = [
"serde",
]
[[package]]
name = "serde"
@ -592,6 +752,47 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.120"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "serde_with"
version = "3.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e73139bc5ec2d45e6c5fd85be5a46949c1c39a4c18e56915f5eb4c12f975e377"
dependencies = [
"base64",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.2.6",
"serde",
"serde_derive",
"serde_json",
"serde_with_macros",
"time",
]
[[package]]
name = "serde_with_macros"
version = "3.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b80d3d6b56b64335c0180e5ffde23b3c5e08c14c585b51a15bd0e95393f46703"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "strsim"
version = "0.11.1"
@ -627,6 +828,37 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
[[package]]
name = "time"
version = "0.3.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
dependencies = [
"deranged",
"itoa",
"num-conv",
"powerfmt",
"serde",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
dependencies = [
"num-conv",
"time-core",
]
[[package]]
name = "unicode-ident"
version = "1.0.12"

View file

@ -11,8 +11,11 @@ env_logger = "0.11.3"
clap-verbosity-flag = "2.2"
anyhow = "1.0"
chrono = { version = "0.4", features = ["serde"] }
semver = "1.0"
semver = { version = "1.0", features = ["serde"] }
regex = "1.10"
serde = { version = "1.0", features = ["derive"] }
serde_with = "3.8"
csv = "1.3"
[dev-dependencies]
assert_cmd = "2.0"

View file

@ -1,5 +1,7 @@
# psb_qaqc
WIP
## For developers
### build / run
```sh
@ -18,6 +20,12 @@ cargo doc --open
```
## TODO
- add more unittests
- test for various cases
- split into muds(maybe as lib crates)
- organize existing feature as a subcommand
- add csv check/validate subcommand
- skew measurement?
- rename bin/project
check `TODO`s in comments.

View file

@ -1,10 +1,9 @@
use core::{panic, str};
use std::{
collections::HashMap,
env,
fmt::Display,
fs::File,
io::{BufRead, BufReader, Write},
io::{BufRead, BufReader},
path::{self, PathBuf},
str::FromStr,
};
@ -15,17 +14,8 @@ use clap::Parser;
use log::{debug, error, info, trace, warn};
use regex::Regex;
use semver::Version;
fn get_default_output_path() -> PathBuf {
PathBuf::from_str("outpu.csv").expect("Error")
}
fn get_default_log_path() -> PathBuf {
let mut path = env::current_exe().unwrap();
path.pop();
path.push("log");
path.push("debug.log");
path
}
use serde::{Deserialize, Serialize};
use serde_with::{serde_as, DisplayFromStr};
/// Parse master jathub logfile for PS Board QAQC and write out to CSV.
#[derive(Parser, Debug)]
@ -145,7 +135,7 @@ impl FromStr for PsbId {
/// Temporary stores raw values.
/// TODO: specify each filed types.(maybe multi values)
#[derive(Debug)]
struct BoardResult {
struct MasterBoardResult {
id: PsbId,
qspif: u8,
qspip: u8,
@ -158,11 +148,12 @@ struct BoardResult {
/// Full result for a single QAQC run from a log file on JATHub master.
#[derive(Debug)]
struct MasterLogResult {
pub struct MasterLogResult {
version: Version,
datetime: DateTime<Utc>,
shifter: String,
board_results: HashMap<Position, BoardResult>,
board_results: HashMap<Position, MasterBoardResult>,
filename: String,
}
/// Get version of shift script.
@ -193,7 +184,7 @@ fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
impl MasterLogResult {
/// Parse log file on master jathub.
fn parse_file(file: impl BufRead) -> Result<MasterLogResult> {
fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
let mut lines = file.lines();
let version = {
@ -322,7 +313,7 @@ impl MasterLogResult {
.context(format!("No board on pos {}", pos))?
.clone();
let result = BoardResult {
let result = MasterBoardResult {
id: psbid,
qspif: parts
.get(1)
@ -370,17 +361,70 @@ impl MasterLogResult {
datetime,
shifter,
board_results,
filename,
})
}
fn write(&self, f: impl Write) -> Result<()> {
todo!()
}
}
fn check_csv_header(f: impl BufRead) -> Result<()> {
// f.lines()
todo!()
/// All information on QAQC stored in the database.
///
/// Everything without `Option` is available from master log.
///
/// TODO: use pos? shifter? version?
#[serde_as]
#[derive(PartialEq, Debug, Serialize, Deserialize)]
pub struct PsbQaqcResult {
motherboard_id: u32,
daughterboard_id: Option<u32>,
#[serde_as(as = "DisplayFromStr")]
position: Position,
qspif: u8,
qspip: u8,
recov: u8,
clock: u8,
regac: u8,
asdtp: u8,
qaqc_result: u32,
lvds_tx_skew: Option<u32>,
ppl_lock_reset_count: Option<i32>,
timestamp: DateTime<Utc>,
qaqc_log_file: String,
firmware_ver: Option<Version>,
parameter_ver: Option<Version>,
fpga_dna: Option<u64>,
comment: String,
}
impl PsbQaqcResult {
/// Expand [`MasterLogResult`] to [`PsbQaqcResult`].
/// Filling unavailable fileds with [`None`]s.
pub fn from_masterlogresult(result: MasterLogResult) -> Vec<Self> {
let mut converted = vec![];
for (pos, boardresult) in result.board_results {
let new = PsbQaqcResult {
motherboard_id: boardresult.id.id,
daughterboard_id: None,
position: pos,
qspif: boardresult.qspif,
qspip: boardresult.qspip,
recov: boardresult.recov,
clock: boardresult.clock,
regac: boardresult.regac,
asdtp: boardresult.asdtp,
qaqc_result: boardresult.done.into(),
lvds_tx_skew: None,
ppl_lock_reset_count: None,
timestamp: result.datetime,
qaqc_log_file: result.filename.clone(),
firmware_ver: None,
parameter_ver: None,
fpga_dna: None,
comment: "".to_string(),
};
converted.push(new);
}
converted
}
}
fn main() -> Result<()> {
@ -397,9 +441,17 @@ fn main() -> Result<()> {
);
let result = {
let file = File::open(args.master_log)?;
let file = File::open(args.master_log.clone())?;
let reader = BufReader::new(file);
MasterLogResult::parse_file(reader)?
MasterLogResult::parse_file(
reader,
args.master_log
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_string(),
)?
};
debug!("{:?}", result);
@ -407,8 +459,36 @@ fn main() -> Result<()> {
// let file = File::options()
// .read(true)
// .open(args.outfile)?;
// }
//
// let mut rdr = csv::Reader::from_reader(file);
// rdr.records();
// }
let expanded_results = PsbQaqcResult::from_masterlogresult(result);
let mut wtr = match args.outfile.exists() {
true => {
let file = File::options().read(true).append(true).open(args.outfile)?;
csv::WriterBuilder::new()
.has_headers(false)
.from_writer(file)
}
false => {
println!("Creating new file: {}", args.outfile.display());
let file = File::options()
.create_new(true)
.write(true)
.open(args.outfile)?;
csv::WriterBuilder::new()
.has_headers(true)
.from_writer(file)
}
};
for result in expanded_results {
wtr.serialize(result)?;
}
wtr.flush()?;
Ok(())
}
@ -417,8 +497,11 @@ mod test {
use std::str::FromStr;
use chrono::{DateTime, Utc};
use semver::Version;
use crate::{extract_position_id, extract_shifter_line, Position, PositionLayer, PsbId};
use crate::{
extract_position_id, extract_shifter_line, extract_version, Position, PositionLayer, PsbId,
};
#[test]
fn parse_position() {
@ -454,6 +537,14 @@ mod test {
assert!(DateTime::parse_from_str("Date: 2024-06-20T08:42:01+0000", "Date: %+").is_ok());
}
#[test]
fn parse_version() {
assert_eq!(
extract_version("Shift script: 1.0.0").unwrap(),
Version::new(1, 0, 0)
)
}
#[test]
fn parse_shifter_line() {
assert_eq!(extract_shifter_line("Shifters: foo").unwrap(), "foo");

View file

@ -15,9 +15,8 @@ mod integrated_test {
.arg("./example_logs/valid/20240620_083537.log")
.arg(test_out.as_path())
.assert()
.success();
// .failure()
// .stderr(predicate::str::contains("not yet implemented"));
.success()
.stdout(predicate::str::contains("Creating new file"));
Ok(())
}
@ -32,9 +31,8 @@ mod integrated_test {
.arg("./example_logs/valid/20240620_093537.log")
.arg(test_out.as_path())
.assert()
.success();
// .failure()
// .stderr(predicate::str::contains("not yet implemented"));
.success()
.stdout(predicate::str::contains("Creating new file"));
Ok(())
}