fix: CHANGELOG diff links

update CHANGELOG
update: print error when no editor is specified & add-master-log doesn't
2025-04-18 18:56:10 +09:00 · 2024-09-11 15:31:09 +09:00 · 2024-09-11 15:25:50 +09:00 · 2024-09-11 15:19:13 +09:00 · 2024-09-11 10:31:28 +09:00 · 2024-09-11 10:17:49 +09:00
11 changed files with 738 additions and 68 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -7,9 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## [Unreleased]

+### Changed
+
+- `add-master-log` now always create CSV file
+- `convert-master-log` can still append to a CSV file
+
+## [2.0.1]
+
+### Fix
+
+- version in Cargo.toml
+
+## [2.0.0]
+
+### Added
+
+- `add-master-log` subcommand provide interactive session to upload master log to database via CSV
+
+### Change
+
+- *BREAKING* old `add-master-log` is now renamed to `convert-master-log`
+
+## [1.1.0]
+
+### Added
+
+- Use `out_<runid>.csv` as default output path
+
+## [1.0.1]
+
+### Fixed
+
+- update test for `retest` field
+- version field in Cargo.toml
+
+### Added
+
+- updated check-db for `abnormal resistance`
+- Docs (for japanese)
+- automatically detect boards to retest
+
+## [1.0.0] - 2024-07-25
+
 ### Added

 - Subcommand `add-master-log` to parse master log for shiftwork 0.1.0 and write out to CSV.
 - Subcommand `check-db` to validate the database CSV file.

-[Unreleased]: https://gitlab.cern.ch
+[Unreleased]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.0.1...main
+[2.0.1]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v2.0.0...v2.0.1
+[2.0.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.1.0...v2.0.0
+[1.1.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.0.1...v1.1.0
+[1.0.1]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.0.0...v1.0.1
+[1.0.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/tags/v1.0.0
--- a/Cargo.lock
+++ b/Cargo.lock
@ -260,6 +260,31 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"

+[[package]]
+name = "crossterm"
+version = "0.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
+dependencies = [
+ "bitflags",
+ "crossterm_winapi",
+ "mio",
+ "parking_lot",
+ "rustix",
+ "signal-hook",
+ "signal-hook-mio",
+ "winapi",
+]
+
+[[package]]
+name = "crossterm_winapi"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "csv"
 version = "1.3.0"
@ -446,6 +471,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"

+[[package]]
+name = "hermit-abi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@ -567,6 +598,16 @@ version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"

+[[package]]
+name = "lock_api"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
 [[package]]
 name = "log"
 version = "0.4.22"
@ -579,6 +620,19 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"

+[[package]]
+name = "mio"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys",
+]
+
 [[package]]
 name = "normalize-line-endings"
 version = "0.3.0"
@ -606,6 +660,29 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"

+[[package]]
+name = "parking_lot"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-targets",
+]
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@ -653,7 +730,7 @@ dependencies = [

 [[package]]
 name = "psb-qaqc"
-version = "0.1.0"
+version = "2.0.1"
 dependencies = [
 "anyhow",
 "assert_cmd",
@ -662,6 +739,7 @@ dependencies = [
 "clap",
 "clap-verbosity-flag",
 "clap_derive",
+ "crossterm",
 "csv",
 "env_logger",
 "itertools",
@ -682,6 +760,15 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "redox_syscall"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "regex"
 version = "1.10.5"
@ -739,6 +826,12 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "semver"
 version = "1.0.23"
@ -809,6 +902,42 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "signal-hook"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-mio"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
+dependencies = [
+ "libc",
+ "mio",
+ "signal-hook",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@ -906,6 +1035,12 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.92"
@ -960,6 +1095,22 @@ version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"

+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
 [[package]]
 name = "winapi-util"
 version = "0.1.8"
@ -969,6 +1120,12 @@ dependencies = [
 "windows-sys",
 ]

+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-core"
 version = "0.52.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -1,6 +1,6 @@
 [package]
 name = "psb-qaqc"
-version = "0.1.0"
+version = "2.0.1"
 edition = "2021"

 [dependencies]
@ -17,6 +17,7 @@ regex = "1.10"
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "3.8"
 csv = "1.3"
+crossterm = "0.28"

 [dev-dependencies]
 assert_cmd = "2.0"
--- a/21
+++ b/21
@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Wataru Otsubo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/README.md
+++ b/README.md
@ -2,7 +2,7 @@

 WIP

-For Japanese: [./docs/README-ja.md](README-ja.md)
+日本語: [README-ja.md](./docs/README-ja.md)

 ## For developers
 Use `rustup` to set up developing tools.
@ -36,5 +36,6 @@ cargo doc --open
 - add csv check/validate subcommand
 - skew measurement?
 - skew measurement test
+- for start-shiftwork v1.0.0 (add start date parsing)

 check `TODO`s in comments.
--- a/docs/README-ja.md
+++ b/docs/README-ja.md
@ -1,22 +1,70 @@
 # psb_qaqc
-日本語ドキュメント(WIP)
+QAQC実施直後にmasterでログをまとめるときに使うコマンドのマニュアル。
+
+## シフトでの使い方
+
+### 1. masterのログをCSVに変換する
+
+```
+# psb-qaqc add-master-log <LOG FILE> <DATABASE CSV>
+```
+
+```sh
+psb-qaqc add-master-log ./logs/20.log out_20.csv
+```
+
+### 2. 再試験にまわすボードを決める
+TODO: 判断基準
+
+masterのログまたはcsvを参照しながら再試験するかを判断し、
+再試験するものはさっき作ったcsvファイルの右側の`false`を`true`に書き換える。
+また、その右側にコメントを書き加える。
+
+[CSVへの書き込みについて](#CSVへの書き込みについて)
+
+### 3. Google sheetsにアップロードする
+CSVをコピーして、[Google sheets](https://docs.google.com/spreadsheets/d/128qOseOy4QDotehYe4Wf2jj88tnwiXGVdR3NHrjcDYU/edit?pli=1&gid=408695746#gid=408695746)の一番下に加える。
+*列をわける*のようなものを選択するとうまく貼れる。
+
+### 4. CSVの後処理
+アップロード済みである(編集しない)ことを明らかにするために、`log`ディレクトリに`_uploaded.csv`の名前にして移動させる。

 ## 機能
 サブコマンドごとに機能がわかれている。
-各コマンドで`--help`を渡せばドキュメントがみれる。
+各コマンドで`--help`を渡せばヘルプがみれる。

 ### `add-master-log`
 QAQC実施直後にMaster JATHub上で実行することでMasterでのログをCSVに書き加える。

-```sh
-psb-qaqc add-master-log ./logs/20240722_180104.log out.csv
+```
+# psb-qaqc add-master-log <LOG FILE> <DATABASE CSV>
 ```

-### `check-db` (WIP)
+```sh
+psb-qaqc add-master-log ./logs/20.log out.csv
+```
+
+### `check-db`
 CSVファイルのフォーマットが正常か確認する。
-正常でない場合、返り値は0以外を返す。
+正常でない場合、0以外を返す。

 ```sh
 psb-qaqc check-db out.csv
 ```

+## CSVへの書き込みについて
+CSVの一番最後の列にコメントを書き込めるようにしている。
+以下コメントを書き込む際の注意点・推奨事項:
+
+- `" "` で囲む
+- `,`は書き込める
+- `"`を使うときは`""`のように重ねてかく
+- 一番うしろには`,`を書かない
+
+*例*
+```csv
+,,"日本語も,かけます ""field"""
+```
+
+編集前にバックアップを作成することを推奨します。
+
--- a/src/main.rs
+++ b/src/main.rs
@ -1,8 +1,9 @@
 use core::str;
 use std::{
+    env,
    fmt::Display,
-    fs::File,
-    io::BufReader,
+    fs::{self, File},
+    io::{self, BufRead, BufReader},
    path::{self, PathBuf},
    str::FromStr,
 };
@ -10,8 +11,9 @@ use std::{
 use anyhow::{anyhow, Context, Result};
 use chrono::{DateTime, Utc};
 use clap::{Parser, Subcommand};
-use log::{debug, trace, warn};
-use masterlog::MasterLogResult;
+use crossterm::{event, terminal};
+use log::{debug, error, info, trace, warn};
+use masterlog::{get_output_filename, MasterLogResult};
 use semver::Version;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
@ -31,13 +33,27 @@ struct Args {

 #[derive(Subcommand, Debug)]
 pub enum Commands {
-    /// Parse master jathub logfile for PS Board QAQC and write out to CSV.
+    /// Parse master log and convert to CSV and prompt post processes interactively.
    AddMasterLog {
        /// Master log file.
        master_log: path::PathBuf,
        /// Output CSV file.
+        /// Default is `out_<runid>.csv`
        // #[arg(default_value = get_default_log_path().into_os_string())]
-        outfile: PathBuf,
+        outfile: Option<PathBuf>,
+        /// Editor to use.
+        /// Default is `$VISUAL` or `$EDITOR`.
+        #[arg(short, long)]
+        editor: Option<String>,
+    },
+    /// Parse master jathub logfile for PS Board QAQC and write out to CSV.
+    ConvertMasterLog {
+        /// Master log file.
+        master_log: path::PathBuf,
+        /// Output CSV file.
+        /// Default is `out_<runid>.csv`
+        // #[arg(default_value = get_default_log_path().into_os_string())]
+        outfile: Option<PathBuf>,
    },
    /// Check CSV format
    CheckDB {
@ -184,6 +200,7 @@ pub struct PsbQaqcResult {
    firmware_ver: Option<Version>,
    parameter_ver: Option<Version>,
    fpga_dna: Option<u64>,
+    retest: bool,
    comment: String,
 }

@ -193,6 +210,7 @@ impl PsbQaqcResult {
    pub fn from_masterlogresult(result: MasterLogResult) -> Vec<Self> {
        let mut converted = vec![];
        for (pos, boardresult) in result.board_results {
+            let isretest = boardresult.is_need_retest().is_some();
            let new = PsbQaqcResult {
                motherboard_id: boardresult.id.id,
                daughterboard_id: None,
@ -213,6 +231,7 @@ impl PsbQaqcResult {
                firmware_ver: None,
                parameter_ver: None,
                fpga_dna: None,
+                retest: isretest,
                comment: "".to_string(),
            };
            converted.push(new);
@ -221,6 +240,154 @@ impl PsbQaqcResult {
    }
 }

+// /// Iterator over output csv file but without results from abnormal resistance
+// #[derive(Debug)]
+// struct CsvMidReader<B> {
+//     lines: std::io::Lines<B>,
+//     /// skipped line numbers
+//     pub skipped: Vec<usize>,
+//     /// Current line count
+//     count: usize,
+//     linebuf: Vec<u8>,
+// }
+//
+// impl<B> CsvMidReader<B> {
+//     pub fn new(lines: std::io::Lines<B>) -> Self {
+//         CsvMidReader {
+//             lines,
+//             skipped: vec![],
+//             count: 0,
+//             linebuf: b"".to_vec(),
+//         }
+//     }
+// }
+//
+// impl<B: BufRead> Read for CsvMidReader<B> {
+//     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+//         let mut rest_buf_size = buf.len();
+//         let mut written_size = 0;
+//         while self.linebuf.len() <= rest_buf_size {
+//             trace!("linebuf: {:?}", self.linebuf.len());
+//             (0..(self.linebuf.len())).for_each(|i| {
+//                 buf[i + written_size] = self.linebuf[i];
+//             });
+//             written_size += self.linebuf.len();
+//             rest_buf_size -= self.linebuf.len();
+//             self.linebuf = match self.lines.next() {
+//                 None => {
+//                     trace!("None");
+//                     break;
+//                 }
+//                 Some(line) => {
+//                     trace!("new line: {:?}", line);
+//                     line?.as_bytes().to_vec()
+//                 }
+//             };
+//         }
+//         if self.lines.next().is_some() {
+//             (0..rest_buf_size).for_each(|i| {
+//                 buf[i + written_size] = self.linebuf[i];
+//             });
+//             self.linebuf.drain(0..rest_buf_size);
+//             written_size += rest_buf_size;
+//         } else {
+//             self.linebuf = b"".to_vec();
+//         }
+//         trace!("size: {}", written_size);
+//         Ok(written_size)
+//     }
+// }
+//
+// impl<B: BufRead> Iterator for CsvMidReader<B> {
+//     type Item = std::result::Result<String, std::io::Error>;
+//
+//     fn next(&mut self) -> Option<Self::Item> {
+//         loop {
+//             match self.lines.next() {
+//                 Some(line) => {
+//                     self.count += 1;
+//                     match line {
+//                         Ok(line) => match line.contains("abnormal resistance") {
+//                             true => {
+//                                 self.skipped.push(self.count);
+//                                 continue;
+//                             }
+//                             false => return Some(Ok(line)),
+//                         },
+//                         Err(e) => return Some(Err(e.into())),
+//                     }
+//                 }
+//                 None => return None,
+//             }
+//         }
+//     }
+// }
+//
+// // impl<B> std::io::Read for CsvMidReader<B> {
+// //     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+// //         todo!()
+// //     }
+// // }
+// //
+// fn filter_csv_lines(rdr: impl BufRead) -> Vec<String> {
+//     rdr.lines()
+//         .enumerate()
+//         .filter(|(num, line)| match line {
+//             Ok(line) => {
+//                 if line.contains("abnormal resistance") {
+//                     info!("Skipping line {}", num + 1);
+//                     debug!("line: {}", line);
+//                     false
+//                 } else {
+//                     true
+//                 }
+//             }
+//             Err(e) => {
+//                 error!("Error while reading: {}", e);
+//                 false
+//             }
+//         })
+//         .map(|(_, line)| line.unwrap())
+//         .collect_vec()
+// }
+
+/// If `nonewfile` is `true`, out file is not created when there is already a file.
+fn write_psbqaqc_csv(result: MasterLogResult, outfile: PathBuf, nonewfile: bool) -> Result<()> {
+    let expanded_results = PsbQaqcResult::from_masterlogresult(result);
+
+    let mut wtr = match (outfile.exists(), nonewfile) {
+        (true, false) => {
+            let file = File::options()
+                .read(true)
+                .append(true)
+                .open(outfile.clone())?;
+            csv::WriterBuilder::new()
+                .has_headers(false)
+                .from_writer(file)
+        }
+        (false, _) => {
+            println!("Creating new file: {}", outfile.display());
+            let file = File::options()
+                .create_new(true)
+                .write(true)
+                .open(outfile.clone())?;
+            csv::WriterBuilder::new()
+                .has_headers(true)
+                .from_writer(file)
+        }
+        (true, true) => {
+            error!("Out file already exists but specified not to make new file");
+            return Err(anyhow!("Out file already exists"));
+        }
+    };
+    for result in expanded_results {
+        wtr.serialize(result)?;
+    }
+    wtr.flush()?;
+
+    Ok(())
+}
+
 fn main() -> Result<()> {
    let args = Args::parse();
    env_logger::Builder::new()
@ -237,58 +404,157 @@ fn main() -> Result<()> {
        Commands::AddMasterLog {
            master_log,
            outfile,
+            editor,
        } => {
-            let result = {
-                let file = File::open(master_log.clone())?;
-                let reader = BufReader::new(file);
-                MasterLogResult::parse_file(
-                    reader,
-                    master_log
-                        .file_name()
-                        .unwrap()
-                        .to_str()
-                        .unwrap()
-                        .to_string(),
-                )?
-            };
+            let result = MasterLogResult::parse_file(master_log)?;
            debug!("{:?}", result);

-            let expanded_results = PsbQaqcResult::from_masterlogresult(result);
+            // Print boards to retest
+            result.print_boards_to_retest(io::stdout())?;

-            let mut wtr = match outfile.exists() {
-                true => {
-                    let file = File::options().read(true).append(true).open(outfile)?;
-                    csv::WriterBuilder::new()
-                        .has_headers(false)
-                        .from_writer(file)
+            let outfile = outfile.unwrap_or(get_output_filename(&result));
+
+            write_psbqaqc_csv(result, outfile.clone(), true)?;
+
+            println!("Add comments about errors in the last column of the CSV file");
+            println!("Press any key to start editting...");
+            terminal::enable_raw_mode()?;
+            let _ = event::read()?;
+            terminal::disable_raw_mode()?;
+
+            let editor = match (editor, env::var("VISUAL"), env::var("EDITOR")) {
+                (Some(editor), _, _) => editor,
+                (None, Ok(editor), _) => {
+                    info!("Use VISUAL");
+                    editor
                }
-                false => {
-                    println!("Creating new file: {}", outfile.display());
-                    let file = File::options().create_new(true).write(true).open(outfile)?;
-                    csv::WriterBuilder::new()
-                        .has_headers(true)
-                        .from_writer(file)
+                (None, Err(_), Ok(editor)) => {
+                    info!("Use EDITOR");
+                    editor
+                }
+                (None, Err(e1), Err(e2)) => {
+                    warn!("No VISUAL nor EDITOR found, {}, {}", e1, e2);
+                    warn!("Using `nano` as a fallback editor");
+                    "nano".to_string()
                }
            };
-            for result in expanded_results {
-                wtr.serialize(result)?;
+            std::process::Command::new(editor)
+                .arg(outfile.clone())
+                .spawn()
+                .context("Error during spawning editor")?
+                .wait()
+                .context("Error while waiting editor")?;
+
+            {
+                let f = File::open(outfile.clone())?;
+                let rdr = BufReader::new(f);
+                rdr.lines().for_each(|l| println!("{}", l.unwrap()));
            }
-            wtr.flush()?;
+            println!();
+            println!("Copy the CSV above and paste it to the database(Google sheets)");
+            println!("Choose `Data->Split text to columns` to format it");
+
+            println!("Press any key when upload finished...");
+            terminal::enable_raw_mode()?;
+            let _ = event::read()?;
+            terminal::disable_raw_mode()?;
+
+            let uploaded_outfile = {
+                let mut new_file_name = outfile
+                    .clone()
+                    .file_stem()
+                    .context("No file stem for out file")?
+                    .to_owned();
+                new_file_name.push("_uploaded");
+                if let Some(ext) = outfile.extension() {
+                    new_file_name.push(".");
+                    new_file_name.push(ext);
+                };
+                PathBuf::from("log").join(new_file_name)
+            };
+            info!(
+                "Renaming {} to {}",
+                outfile.display(),
+                uploaded_outfile.is_dir()
+            );
+            fs::rename(outfile, uploaded_outfile)?;
+        }
+        Commands::ConvertMasterLog {
+            master_log,
+            outfile,
+        } => {
+            let result = MasterLogResult::parse_file(master_log)?;
+            debug!("{:?}", result);
+
+            // Print boards to retest
+            result.print_boards_to_retest(io::stdout())?;
+
+            let outfile = outfile.unwrap_or(get_output_filename(&result));
+
+            write_psbqaqc_csv(result, outfile, false)?;
        }
        Commands::CheckDB { csvfile } => {
            // TODO: more friendly message (like row number)
            // TODO: more validation options (e.g. for each stage(Master log, Slave log(end of
            // campaign)))
            // TODO: Masterログが得られたときや、行として完成していかないときなど複数段階のチェックを用意する
-            let file = File::options().read(true).open(csvfile)?;
+            let file = File::options().read(true).open(csvfile.clone())?;

-            let rdr = csv::Reader::from_reader(file);
-            if !rdr
-                .into_deserialize::<PsbQaqcResult>()
-                .all(|row| row.is_ok())
-            {
-                return Err(anyhow!("Invalid csv"));
+            // tried to implement lazy evaluated `Read`er
+            // let bufrdr = BufReader::new(file);
+            // let skipped_reader = CsvMidReader::new(bufrdr.lines());
+            // let rdr = csv::Reader::from_reader(skipped_reader);
+
+            // filter and collect source beforehand
+            // let rdr = BufReader::new(file);
+            // let nl = "\n";
+            // let content = filter_csv_lines(rdr).join(nl);
+            // let content = content.as_bytes();
+            // let rdr = csv::Reader::from_reader(content);
+
+            let mut file_for_postfilter = {
+                let file = File::options().read(true).open(csvfile)?;
+                BufReader::new(file).lines()
            };
+            let mut prev_postifilter_line = 0;
+            let rdr = csv::Reader::from_reader(file);
+            let mut isinvalid = false;
+            for (num, line) in rdr.into_deserialize::<PsbQaqcResult>().enumerate() {
+                let line = match line {
+                    Ok(line) => Ok(line),
+                    Err(err) => {
+                        // catch
+                        // line num is +1 because csv iterator doesn't include header line
+                        match file_for_postfilter.nth(num + 1 - prev_postifilter_line) {
+                            None => {
+                                warn!("Error while reading file(line: {})", num + 1);
+                                debug!("lines: {:?}", file_for_postfilter);
+                                Err(err)
+                            }
+                            Some(line) => {
+                                prev_postifilter_line = num + 2;
+                                if let Ok(line) = line {
+                                    info!("error line on {}: line: {}", num, line);
+                                    if line.contains("abnormal resistance") {
+                                        println!("\"abnormal resistance\" at line {}", num + 1);
+                                        continue;
+                                    }
+                                }
+                                Err(err)
+                            }
+                        }
+                    }
+                };
+                if line.is_err() {
+                    isinvalid = true;
+                }
+                // increment the following num for *header line* + *0-origin offset*
+                let line = line.inspect_err(|e| error!("Invalid line at {}: {}", num + 2, e));
+                debug!("{}: line {:?}", num, line);
+            }
+            if isinvalid {
+                return Err(anyhow!("Invalid CSV format."));
+            }
        }
        Commands::AddSkew { logfile, outfile } => {
            let skew = skew::parse_count_file(logfile).context("Error during getting skew")?;
--- a/src/masterlog.rs
+++ b/src/masterlog.rs
@ -1,4 +1,10 @@
-use std::{collections::BTreeMap, io::BufRead, str::FromStr};
+use std::{
+    collections::BTreeMap,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    path::PathBuf,
+    str::FromStr,
+};

 use anyhow::{anyhow, Context, Result};
 use chrono::{DateTime, Utc};
@ -6,7 +12,7 @@ use log::{debug, info, trace};
 use regex::Regex;
 use semver::Version;

-use crate::{Position, PositionLayer, PsbId};
+use crate::{Position, PositionLayer, PsbId, PsbQaqcResult};

 /// QAQC results for each boards extracted from master log.
 ///
@ -24,6 +30,36 @@ pub struct MasterBoardResult {
    pub result: u8,
 }

+impl MasterBoardResult {
+    /// Check the board needs to be retested with reason.
+    ///
+    /// None if it doesn't need retest.
+    pub fn is_need_retest(&self) -> Option<String> {
+        let mut result = false;
+        let mut reasons = vec![];
+        if self.qspip != 1 {
+            result = true;
+            reasons.push(format!("QSPIp is not 1({})", self.qspip));
+        }
+        if self.recov != 1 {
+            result = true;
+            reasons.push(format!("Recov is not 1({})", self.recov));
+        }
+        if self.power != 1 {
+            result = true;
+            reasons.push(format!("Power is not 1({})", self.qspip));
+        }
+        if self.clock != 1 {
+            result = true;
+            reasons.push(format!("Clock is not 1({})", self.clock));
+        }
+        match result {
+            true => Some(reasons.join(", ")),
+            false => None,
+        }
+    }
+}
+
 /// Full result for a single QAQC run from a log file on JATHub master.
 #[derive(Debug)]
 pub struct MasterLogResult {
@ -72,7 +108,21 @@ fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {

 impl MasterLogResult {
    /// Parse log file on master jathub.
-    pub fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
+    pub fn parse_file(master_log: PathBuf) -> Result<MasterLogResult> {
+        let file = File::open(master_log.clone())?;
+        let reader = BufReader::new(file);
+        MasterLogResult::parse_file_with_filename(
+            reader,
+            master_log
+                .file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .to_string(),
+        )
+    }
+
+    fn parse_file_with_filename(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
        let mut lines = file.lines();

        let version = {
@ -271,6 +321,23 @@ impl MasterLogResult {
            filename,
        })
    }
+
+    pub fn print_boards_to_retest(&self, mut wtr: impl Write) -> Result<()> {
+        for (pos, boardresult) in &self.board_results {
+            if let Some(reason) = boardresult.is_need_retest() {
+                writeln!(
+                    wtr,
+                    "Board {} at {} need retest for {}",
+                    boardresult.id.id, pos, reason
+                )?;
+            }
+        }
+        Ok(())
+    }
+}
+
+pub fn get_output_filename(result: &MasterLogResult) -> PathBuf {
+    PathBuf::from(format!("out_{}.csv", result.runid))
 }

 #[cfg(test)]
--- a/src/skew.rs
+++ b/src/skew.rs
@ -67,7 +67,7 @@ mod test {
    fn test_get_psbid_from_clklog() {
        assert_eq!(
            get_psbid_from_clklogfile("120_1234.log".into()).unwrap(),
-            (PsbId::new(1020), 1234)
+            (PsbId::new(120), 1234)
        );
        assert_eq!(
            get_psbid_from_clklogfile("1024_124.log".into()).unwrap(),
--- a/tests/cli.rs
+++ b/tests/cli.rs
@ -1,11 +1,13 @@
 mod integrated_test {
    use std::{
-        fs::File,
+        env::current_dir,
+        ffi::OsString,
+        fs::{self, read_dir, File},
        io::{BufRead, BufReader},
        path::PathBuf,
    };

-    use anyhow::Result;
+    use anyhow::{Context, Result};
    use assert_cmd::Command;
    use itertools::Itertools;
    use predicates::prelude::*;
@ -18,8 +20,8 @@ mod integrated_test {
        // 1st file
        let mut cmd = Command::cargo_bin("psb-qaqc")?;
        cmd.current_dir("tests")
-            .arg("add-master-log")
-            .arg("./example_logs/valid/7.log")
+            .arg("convert-master-log")
+            .arg("./example_logs/valid/44.log")
            .arg(test_out.as_path())
            .assert()
            .success()
@ -31,8 +33,8 @@ mod integrated_test {
            let f = File::open(test_out.clone())?;
            let r = BufReader::new(f);
            assert!(r.lines().any(|line| {
-                line.unwrap().contains(
-                    "8866,,B-0-1,0,1,1,0,1,8,1,,7,2024-07-20T17:15:46Z,7.log,0.1.0,alice,,,,",
+                line.unwrap().eq(
+                    "214,,B-0-1,1,1,1,1,1,0,1,,44,2024-07-25T08:41:27Z,44.log,1.0.1,Bob,,,,false,",
                )
            }));
        }
@ -47,7 +49,7 @@ mod integrated_test {
        // 2nd file
        let mut cmd = Command::cargo_bin("psb-qaqc")?;
        cmd.current_dir("tests")
-            .arg("add-master-log")
+            .arg("convert-master-log")
            .arg("./example_logs/valid/20.log")
            .arg(test_out.as_path())
            .assert()
@ -67,17 +69,28 @@ mod integrated_test {
    #[test]
    fn partial_log() -> Result<()> {
        let test_out_dir = assert_fs::TempDir::new()?;
-        let test_out = PathBuf::new().join(&test_out_dir).join("out.csv");
+        fs::copy(
+            "./tests/example_logs/valid/84.log",
+            test_out_dir.join("84.log"),
+        )
+        .context("Error preparing source file")?;

        let mut cmd = Command::cargo_bin("psb-qaqc")?;
-        cmd.current_dir("tests")
-            .arg("add-master-log")
-            .arg("./example_logs/valid/84.log")
-            .arg(test_out.as_path())
+        cmd.current_dir(&test_out_dir)
+            .arg("convert-master-log")
+            .arg("84.log")
            .assert()
            .success()
            .stdout(predicate::str::contains("Creating new file"));

+        assert!(read_dir(&test_out_dir)
+            .context("Failed to read dir")?
+            .any(|entry| {
+                entry
+                    .map_or(OsString::from(""), |entry| entry.file_name())
+                    .eq("out_84.csv")
+            }));
+
        Ok(())
    }
 }
--- a/tests/example_logs/valid/44.log
+++ b/tests/example_logs/valid/44.log
@ -0,0 +1,49 @@
+Shift script: 1.0.1
+----------------------
+Date: 2024-07-25T08:41:27+0000
+QAQC runid: 44
+Shifters: Bob
+----------------------
+PBS Assignment: 
+Position / assigned-ID : B-0-1 / PS000214
+Position / assigned-ID : B-1-1 / PS000215
+Position / assigned-ID : B-0-2 / PS000213
+Position / assigned-ID : B-1-2 / PS000211
+Position / assigned-ID : B-0-3 / PS000212
+Position / assigned-ID : B-1-3 / PS000210
+Position / assigned-ID : B-0-4 / PS000207
+Position / assigned-ID : B-1-4 / PS000209
+Position / assigned-ID : B-0-5 / PS000208
+Position / assigned-ID : B-1-5 / PS000206
+Position / assigned-ID : B-0-6 / PS000205
+Position / assigned-ID : B-1-6 / PS000204
+Position / assigned-ID : B-0-7 / PS000203
+Position / assigned-ID : B-1-7 / PS000202
+Position / assigned-ID : B-0-8 / PS000201
+Position / assigned-ID : B-1-8 / PS000239
+Position / assigned-ID : B-0-9 / PS000238
+Position / assigned-ID : B-1-9 / PS000237
+======================
+    QAQC status| QSPIp | Recov | Power | Clock | ASDTP | Reset | Result |
+    ----------------------------------------------------------------------
+      Station0
+      JATHub_ 1|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_ 2|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_ 3|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_ 4|      1|      1|      1|      2|      1|      4|       2|
+      JATHub_ 5|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_ 6|      1|      1|      1|      1|      1|      1|       1|
+      JATHub_ 7|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_ 8|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_ 9|      1|      1|      1|      2|      1|      0|       2|
+      Station1
+      JATHub_11|      2|      1|      2|      1|      2|      1|       2|
+      JATHub_12|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_13|      1|      1|      1|      2|      1|      0|       2|
+      JATHub_14|      1|      1|      1|      2|      1|      0|       2|
+      JATHub_15|      1|      1|      1|      2|      1|      0|       2|
+      JATHub_16|      1|      1|      2|      1|      1|      1|       2|
+      JATHub_17|      1|      1|      1|      1|      1|      0|       1|
+      JATHub_18|      1|      1|      1|      1|      1|      3|       1|
+      JATHub_19|      1|      1|      1|      1|      1|      0|       1|
+======================
Author	SHA1	Message	Date
Wataru Otsubo	a78416a18d	fix: CHANGELOG diff links	2024-09-11 15:31:09 +09:00
Wataru Otsubo	2dd0c20ce4	update CHANGELOG	2024-09-11 15:25:50 +09:00
Wataru Otsubo	3fdbe5ad4c	update: print error when no editor is specified & add-master-log doesn't append csv file	2024-09-11 15:19:13 +09:00
Wataru Otsubo	4478ec1654	bump to v2.0.1 fix version in Cargo.toml	2024-09-11 10:31:28 +09:00
Wataru Otsubo	6842097962	bump to v2.0.0	2024-09-11 10:17:49 +09:00
Wataru Otsubo	c8525b2ec6	Add LICENSE	2024-09-11 03:04:52 +02:00
Wataru Otsubo	a4c0ac9c55	Merge branch 'feature-more-postprocess' into 'main' [NEW] add more postprocesses to `add-master-log` and rename current one as `convert-master-log` See merge request wotsubo/psboard-qaqc-postprocess!1	2024-09-11 03:01:51 +02:00
Wataru Otsubo	f9716bcc93	update CHANGELOG for add-master-log	2024-09-10 23:04:48 +09:00
Wataru Otsubo	2dc790f9a8	new: add interactive command to add master log to database - old add-master-log is now convert-master-log - add-master-log is now interactive - add-master-log is not well tested	2024-09-10 23:01:20 +09:00
Wataru Otsubo	7fe5218d39	refactor: split csv writing to func	2024-09-10 21:52:41 +09:00
Wataru Otsubo	7529d003a6	refactor: replace MasterLogResult file parse API to use only filename * old one moved to private and used in the new one * for future convert-master-log replacement	2024-09-10 21:22:25 +09:00
Wataru Otsubo	9e93a085fd	refactor: split func for printing boards for retest	2024-09-10 21:07:50 +09:00
Wataru Otsubo	b80c4902ca	bump to 1.1.0	2024-09-10 17:04:57 +09:00
Wataru Otsubo	772f7a155f	added: use out_runid.csv as default output path	2024-07-25 23:34:27 +09:00
Wataru Otsubo	1ec3f8107c	bump to 1.0.1 update CHANGELOG	2024-07-25 20:13:06 +09:00
Wataru Otsubo	ae29be2daa	update: auto distinguish boards to retest - automatically print	2024-07-25 20:07:28 +09:00
Wataru Otsubo	79e7291031	fix: links in README	2024-07-25 13:43:00 +09:00
Wataru Otsubo	d850ef7834	update: doc for Japanese	2024-07-25 13:40:03 +09:00
Wataru Otsubo	46487320b0	update: accept `abnormal resistance` as valid row in check-db - this commits includes 2 other implementations - 1st one with custom type with `impl Read` for Lazy evaluation - this somehow ended up in infinite loop - 2nd one which collects lines without skipped lines first - this works - the current implementations has another File and catches invalid row and check the line later	2024-07-25 13:36:20 +09:00