From 46487320b0027f404aca3c8b5bf9a19e0387a2cd Mon Sep 17 00:00:00 2001 From: Wataru Otsubo Date: Thu, 25 Jul 2024 13:36:20 +0900 Subject: [PATCH] update: accept `abnormal resistance` as valid row in check-db - this commits includes 2 other implementations - 1st one with custom type with `impl Read` for Lazy evaluation - this somehow ended up in infinite loop - 2nd one which collects lines without skipped lines first - this works - the current implementations has another File and catches invalid row and check the line later --- CHANGELOG.md | 14 +++- README.md | 3 +- src/main.rs | 179 ++++++++++++++++++++++++++++++++++++++++++++++++--- src/skew.rs | 2 +- tests/cli.rs | 4 +- 5 files changed, 188 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1744e95..010c2cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- update test for `retest` field + +### Added + +- updated check-db for `abnormal resistance` +- Docs (for japanese) + +## [1.0.0] - 2024-07-25 + ### Added - Subcommand `add-master-log` to parse master log for shiftwork 0.1.0 and write out to CSV. - Subcommand `check-db` to validate the database CSV file. -[Unreleased]: https://gitlab.cern.ch +[Unreleased]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/main...v1.0.0 +[1.0.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/tags/v1.0.0 diff --git a/README.md b/README.md index e168593..ad89d6b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ WIP -For Japanese: [./docs/README-ja.md](README-ja.md) +日本語: [./docs/README-ja.md](README-ja.md) ## For developers Use `rustup` to set up developing tools. @@ -36,5 +36,6 @@ cargo doc --open - add csv check/validate subcommand - skew measurement? - skew measurement test +- for start-shiftwork v1.0.0 (add start date parsing) check `TODO`s in comments. diff --git a/src/main.rs b/src/main.rs index 791a66e..156a5e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,7 @@ use core::str; use std::{ fmt::Display, fs::File, - io::BufReader, + io::{BufRead, BufReader}, path::{self, PathBuf}, str::FromStr, }; @@ -10,7 +10,7 @@ use std::{ use anyhow::{anyhow, Context, Result}; use chrono::{DateTime, Utc}; use clap::{Parser, Subcommand}; -use log::{debug, trace, warn}; +use log::{debug, error, info, trace, warn}; use masterlog::MasterLogResult; use semver::Version; use serde::{Deserialize, Serialize}; @@ -184,6 +184,7 @@ pub struct PsbQaqcResult { firmware_ver: Option, parameter_ver: Option, fpga_dna: Option, + retest: bool, comment: String, } @@ -213,6 +214,7 @@ impl PsbQaqcResult { firmware_ver: None, parameter_ver: None, fpga_dna: None, + retest: false, comment: "".to_string(), }; converted.push(new); @@ -221,6 +223,117 @@ impl PsbQaqcResult { } } +// /// Iterator over output csv file but without results from abnormal resistance +// #[derive(Debug)] +// struct CsvMidReader { +// lines: std::io::Lines, +// /// skipped line numbers +// pub skipped: Vec, +// /// Current line count +// count: usize, +// linebuf: Vec, +// } +// +// impl CsvMidReader { +// pub fn new(lines: std::io::Lines) -> Self { +// CsvMidReader { +// lines, +// skipped: vec![], +// count: 0, +// linebuf: b"".to_vec(), +// } +// } +// } +// +// impl Read for CsvMidReader { +// fn read(&mut self, buf: &mut [u8]) -> std::io::Result { +// let mut rest_buf_size = buf.len(); +// let mut written_size = 0; +// while self.linebuf.len() <= rest_buf_size { +// trace!("linebuf: {:?}", self.linebuf.len()); +// (0..(self.linebuf.len())).for_each(|i| { +// buf[i + written_size] = self.linebuf[i]; +// }); +// written_size += self.linebuf.len(); +// rest_buf_size -= self.linebuf.len(); +// self.linebuf = match self.lines.next() { +// None => { +// trace!("None"); +// break; +// } +// Some(line) => { +// trace!("new line: {:?}", line); +// line?.as_bytes().to_vec() +// } +// }; +// } +// if self.lines.next().is_some() { +// (0..rest_buf_size).for_each(|i| { +// buf[i + written_size] = self.linebuf[i]; +// }); +// self.linebuf.drain(0..rest_buf_size); +// written_size += rest_buf_size; +// } else { +// self.linebuf = b"".to_vec(); +// } +// trace!("size: {}", written_size); +// Ok(written_size) +// } +// } +// +// impl Iterator for CsvMidReader { +// type Item = std::result::Result; +// +// fn next(&mut self) -> Option { +// loop { +// match self.lines.next() { +// Some(line) => { +// self.count += 1; +// match line { +// Ok(line) => match line.contains("abnormal resistance") { +// true => { +// self.skipped.push(self.count); +// continue; +// } +// false => return Some(Ok(line)), +// }, +// Err(e) => return Some(Err(e.into())), +// } +// } +// None => return None, +// } +// } +// } +// } +// +// // impl std::io::Read for CsvMidReader { +// // fn read(&mut self, buf: &mut [u8]) -> std::io::Result { +// // todo!() +// // } +// // } +// // +// fn filter_csv_lines(rdr: impl BufRead) -> Vec { +// rdr.lines() +// .enumerate() +// .filter(|(num, line)| match line { +// Ok(line) => { +// if line.contains("abnormal resistance") { +// info!("Skipping line {}", num + 1); +// debug!("line: {}", line); +// false +// } else { +// true +// } +// } +// Err(e) => { +// error!("Error while reading: {}", e); +// false +// } +// }) +// .map(|(_, line)| line.unwrap()) +// .collect_vec() +// } + fn main() -> Result<()> { let args = Args::parse(); env_logger::Builder::new() @@ -280,15 +393,63 @@ fn main() -> Result<()> { // TODO: more validation options (e.g. for each stage(Master log, Slave log(end of // campaign))) // TODO: Masterログが得られたときや、行として完成していかないときなど複数段階のチェックを用意する - let file = File::options().read(true).open(csvfile)?; + let file = File::options().read(true).open(csvfile.clone())?; - let rdr = csv::Reader::from_reader(file); - if !rdr - .into_deserialize::() - .all(|row| row.is_ok()) - { - return Err(anyhow!("Invalid csv")); + // tried to implement lazy evaluated `Read`er + // let bufrdr = BufReader::new(file); + // let skipped_reader = CsvMidReader::new(bufrdr.lines()); + // let rdr = csv::Reader::from_reader(skipped_reader); + + // filter and collect source beforehand + // let rdr = BufReader::new(file); + // let nl = "\n"; + // let content = filter_csv_lines(rdr).join(nl); + // let content = content.as_bytes(); + // let rdr = csv::Reader::from_reader(content); + + let mut file_for_postfilter = { + let file = File::options().read(true).open(csvfile)?; + BufReader::new(file).lines() }; + let mut prev_postifilter_line = 0; + let rdr = csv::Reader::from_reader(file); + let mut isinvalid = false; + for (num, line) in rdr.into_deserialize::().enumerate() { + let line = match line { + Ok(line) => Ok(line), + Err(err) => { + // catch + // line num is +1 because csv iterator doesn't include header line + match file_for_postfilter.nth(num + 1 - prev_postifilter_line) { + None => { + warn!("Error while reading file(line: {})", num + 1); + debug!("lines: {:?}", file_for_postfilter); + Err(err) + } + Some(line) => { + prev_postifilter_line = num + 2; + if let Ok(line) = line { + info!("error line on {}: line: {}", num, line); + if line.contains("abnormal resistance") { + println!("\"abnormal resistance\" at line {}", num + 1); + continue; + } + } + Err(err) + } + } + } + }; + if line.is_err() { + isinvalid = true; + } + // increment the following num for *header line* + *0-origin offset* + let line = line.inspect_err(|e| error!("Invalid line at {}: {}", num + 2, e)); + debug!("{}: line {:?}", num, line); + } + if isinvalid { + return Err(anyhow!("Invalid CSV format.")); + } } Commands::AddSkew { logfile, outfile } => { let skew = skew::parse_count_file(logfile).context("Error during getting skew")?; diff --git a/src/skew.rs b/src/skew.rs index 73bf381..e586610 100644 --- a/src/skew.rs +++ b/src/skew.rs @@ -67,7 +67,7 @@ mod test { fn test_get_psbid_from_clklog() { assert_eq!( get_psbid_from_clklogfile("120_1234.log".into()).unwrap(), - (PsbId::new(1020), 1234) + (PsbId::new(120), 1234) ); assert_eq!( get_psbid_from_clklogfile("1024_124.log".into()).unwrap(), diff --git a/tests/cli.rs b/tests/cli.rs index 07ff4de..4f7dba0 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -31,8 +31,8 @@ mod integrated_test { let f = File::open(test_out.clone())?; let r = BufReader::new(f); assert!(r.lines().any(|line| { - line.unwrap().contains( - "8866,,B-0-1,0,1,1,0,1,8,1,,7,2024-07-20T17:15:46Z,7.log,0.1.0,alice,,,,", + line.unwrap().eq( + "8866,,B-0-1,0,1,1,0,1,8,1,,7,2024-07-20T17:15:46Z,7.log,0.1.0,alice,,,,false,", ) })); }