update: accept abnormal resistance as valid row in check-db

- this commits includes 2 other implementations
  - 1st one with custom type with `impl Read` for Lazy evaluation
    - this somehow ended up in infinite loop
  - 2nd one which collects lines without skipped lines first
    - this works
- the current implementations has another File and catches invalid row
  and check the line later

CHANGELOG.md

@@ -7,9 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- update test for `retest` field
+
+### Added
+
+- updated check-db for `abnormal resistance`
+- Docs (for japanese)
+
+## [1.0.0] - 2024-07-25
+
 ### Added
 
 - Subcommand `add-master-log` to parse master log for shiftwork 0.1.0 and write out to CSV.
 - Subcommand `check-db` to validate the database CSV file.
 
-[Unreleased]: https://gitlab.cern.ch
+[Unreleased]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/main...v1.0.0
+[1.0.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/tags/v1.0.0

README.md

@@ -2,7 +2,7 @@
 
 WIP
 
-For Japanese: [./docs/README-ja.md](README-ja.md)
+日本語: [./docs/README-ja.md](README-ja.md)
 
 ## For developers
 Use `rustup` to set up developing tools.
@@ -36,5 +36,6 @@ cargo doc --open
 - add csv check/validate subcommand
 - skew measurement?
 - skew measurement test
+- for start-shiftwork v1.0.0 (add start date parsing)
 
 check `TODO`s in comments.

src/main.rs

@@ -2,7 +2,7 @@ use core::str;
 use std::{
     fmt::Display,
     fs::File,
-    io::BufReader,
+    io::{BufRead, BufReader},
     path::{self, PathBuf},
     str::FromStr,
 };
@@ -10,7 +10,7 @@ use std::{
 use anyhow::{anyhow, Context, Result};
 use chrono::{DateTime, Utc};
 use clap::{Parser, Subcommand};
-use log::{debug, trace, warn};
+use log::{debug, error, info, trace, warn};
 use masterlog::MasterLogResult;
 use semver::Version;
 use serde::{Deserialize, Serialize};
@@ -184,6 +184,7 @@ pub struct PsbQaqcResult {
     firmware_ver: Option<Version>,
     parameter_ver: Option<Version>,
     fpga_dna: Option<u64>,
+    retest: bool,
     comment: String,
 }
 
@@ -213,6 +214,7 @@ impl PsbQaqcResult {
                 firmware_ver: None,
                 parameter_ver: None,
                 fpga_dna: None,
+                retest: false,
                 comment: "".to_string(),
             };
             converted.push(new);
@@ -221,6 +223,117 @@ impl PsbQaqcResult {
     }
 }
 
+// /// Iterator over output csv file but without results from abnormal resistance
+// #[derive(Debug)]
+// struct CsvMidReader<B> {
+//     lines: std::io::Lines<B>,
+//     /// skipped line numbers
+//     pub skipped: Vec<usize>,
+//     /// Current line count
+//     count: usize,
+//     linebuf: Vec<u8>,
+// }
+//
+// impl<B> CsvMidReader<B> {
+//     pub fn new(lines: std::io::Lines<B>) -> Self {
+//         CsvMidReader {
+//             lines,
+//             skipped: vec![],
+//             count: 0,
+//             linebuf: b"".to_vec(),
+//         }
+//     }
+// }
+//
+// impl<B: BufRead> Read for CsvMidReader<B> {
+//     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+//         let mut rest_buf_size = buf.len();
+//         let mut written_size = 0;
+//         while self.linebuf.len() <= rest_buf_size {
+//             trace!("linebuf: {:?}", self.linebuf.len());
+//             (0..(self.linebuf.len())).for_each(|i| {
+//                 buf[i + written_size] = self.linebuf[i];
+//             });
+//             written_size += self.linebuf.len();
+//             rest_buf_size -= self.linebuf.len();
+//             self.linebuf = match self.lines.next() {
+//                 None => {
+//                     trace!("None");
+//                     break;
+//                 }
+//                 Some(line) => {
+//                     trace!("new line: {:?}", line);
+//                     line?.as_bytes().to_vec()
+//                 }
+//             };
+//         }
+//         if self.lines.next().is_some() {
+//             (0..rest_buf_size).for_each(|i| {
+//                 buf[i + written_size] = self.linebuf[i];
+//             });
+//             self.linebuf.drain(0..rest_buf_size);
+//             written_size += rest_buf_size;
+//         } else {
+//             self.linebuf = b"".to_vec();
+//         }
+//         trace!("size: {}", written_size);
+//         Ok(written_size)
+//     }
+// }
+//
+// impl<B: BufRead> Iterator for CsvMidReader<B> {
+//     type Item = std::result::Result<String, std::io::Error>;
+//
+//     fn next(&mut self) -> Option<Self::Item> {
+//         loop {
+//             match self.lines.next() {
+//                 Some(line) => {
+//                     self.count += 1;
+//                     match line {
+//                         Ok(line) => match line.contains("abnormal resistance") {
+//                             true => {
+//                                 self.skipped.push(self.count);
+//                                 continue;
+//                             }
+//                             false => return Some(Ok(line)),
+//                         },
+//                         Err(e) => return Some(Err(e.into())),
+//                     }
+//                 }
+//                 None => return None,
+//             }
+//         }
+//     }
+// }
+//
+// // impl<B> std::io::Read for CsvMidReader<B> {
+// //     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+// //         todo!()
+// //     }
+// // }
+// //
+// fn filter_csv_lines(rdr: impl BufRead) -> Vec<String> {
+//     rdr.lines()
+//         .enumerate()
+//         .filter(|(num, line)| match line {
+//             Ok(line) => {
+//                 if line.contains("abnormal resistance") {
+//                     info!("Skipping line {}", num + 1);
+//                     debug!("line: {}", line);
+//                     false
+//                 } else {
+//                     true
+//                 }
+//             }
+//             Err(e) => {
+//                 error!("Error while reading: {}", e);
+//                 false
+//             }
+//         })
+//         .map(|(_, line)| line.unwrap())
+//         .collect_vec()
+// }
+
 fn main() -> Result<()> {
     let args = Args::parse();
     env_logger::Builder::new()
@@ -280,15 +393,63 @@ fn main() -> Result<()> {
             // TODO: more validation options (e.g. for each stage(Master log, Slave log(end of
             // campaign)))
             // TODO: Masterログが得られたときや、行として完成していかないときなど複数段階のチェックを用意する
-            let file = File::options().read(true).open(csvfile)?;
+            let file = File::options().read(true).open(csvfile.clone())?;
 
-            let rdr = csv::Reader::from_reader(file);
+            // tried to implement lazy evaluated `Read`er
-            if !rdr
+            // let bufrdr = BufReader::new(file);
-                .into_deserialize::<PsbQaqcResult>()
+            // let skipped_reader = CsvMidReader::new(bufrdr.lines());
-                .all(|row| row.is_ok())
+            // let rdr = csv::Reader::from_reader(skipped_reader);
-            {
+
-                return Err(anyhow!("Invalid csv"));
+            // filter and collect source beforehand
+            // let rdr = BufReader::new(file);
+            // let nl = "\n";
+            // let content = filter_csv_lines(rdr).join(nl);
+            // let content = content.as_bytes();
+            // let rdr = csv::Reader::from_reader(content);
+
+            let mut file_for_postfilter = {
+                let file = File::options().read(true).open(csvfile)?;
+                BufReader::new(file).lines()
             };
+            let mut prev_postifilter_line = 0;
+            let rdr = csv::Reader::from_reader(file);
+            let mut isinvalid = false;
+            for (num, line) in rdr.into_deserialize::<PsbQaqcResult>().enumerate() {
+                let line = match line {
+                    Ok(line) => Ok(line),
+                    Err(err) => {
+                        // catch
+                        // line num is +1 because csv iterator doesn't include header line
+                        match file_for_postfilter.nth(num + 1 - prev_postifilter_line) {
+                            None => {
+                                warn!("Error while reading file(line: {})", num + 1);
+                                debug!("lines: {:?}", file_for_postfilter);
+                                Err(err)
+                            }
+                            Some(line) => {
+                                prev_postifilter_line = num + 2;
+                                if let Ok(line) = line {
+                                    info!("error line on {}: line: {}", num, line);
+                                    if line.contains("abnormal resistance") {
+                                        println!("\"abnormal resistance\" at line {}", num + 1);
+                                        continue;
+                                    }
+                                }
+                                Err(err)
+                            }
+                        }
+                    }
+                };
+                if line.is_err() {
+                    isinvalid = true;
+                }
+                // increment the following num for *header line* + *0-origin offset*
+                let line = line.inspect_err(|e| error!("Invalid line at {}: {}", num + 2, e));
+                debug!("{}: line {:?}", num, line);
+            }
+            if isinvalid {
+                return Err(anyhow!("Invalid CSV format."));
+            }
         }
         Commands::AddSkew { logfile, outfile } => {
             let skew = skew::parse_count_file(logfile).context("Error during getting skew")?;

src/skew.rs

@@ -67,7 +67,7 @@ mod test {
     fn test_get_psbid_from_clklog() {
         assert_eq!(
             get_psbid_from_clklogfile("120_1234.log".into()).unwrap(),
-            (PsbId::new(1020), 1234)
+            (PsbId::new(120), 1234)
         );
         assert_eq!(
             get_psbid_from_clklogfile("1024_124.log".into()).unwrap(),

tests/cli.rs

@@ -31,8 +31,8 @@ mod integrated_test {
             let f = File::open(test_out.clone())?;
             let r = BufReader::new(f);
             assert!(r.lines().any(|line| {
-                line.unwrap().contains(
+                line.unwrap().eq(
-                    "8866,,B-0-1,0,1,1,0,1,8,1,,7,2024-07-20T17:15:46Z,7.log,0.1.0,alice,,,,",
+                    "8866,,B-0-1,0,1,1,0,1,8,1,,7,2024-07-20T17:15:46Z,7.log,0.1.0,alice,,,,false,",
                 )
             }));
         }