Merge branch 'feature-more-postprocess' into 'main'

[NEW] add more postprocesses to `add-master-log` and rename current one as `convert-master-log`

See merge request wotsubo/psboard-qaqc-postprocess!1

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `add-master-log` subcommand provide interactive session to upload master log to database via CSV
+
+### Change
+
+- *BREAKING* old `add-master-log` is now renamed to `convert-master-log`
+
 ## [1.1.0]
 
 ### Added

Cargo.lock

@@ -260,6 +260,31 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
+[[package]]
+name = "crossterm"
+version = "0.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
+dependencies = [
+ "bitflags",
+ "crossterm_winapi",
+ "mio",
+ "parking_lot",
+ "rustix",
+ "signal-hook",
+ "signal-hook-mio",
+ "winapi",
+]
+
+[[package]]
+name = "crossterm_winapi"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "csv"
 version = "1.3.0"
@@ -446,6 +471,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -567,6 +598,16 @@ version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
 
+[[package]]
+name = "lock_api"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
 [[package]]
 name = "log"
 version = "0.4.22"
@@ -579,6 +620,19 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "mio"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys",
+]
+
 [[package]]
 name = "normalize-line-endings"
 version = "0.3.0"
@@ -606,6 +660,29 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "parking_lot"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-targets",
+]
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -662,6 +739,7 @@ dependencies = [
  "clap",
  "clap-verbosity-flag",
  "clap_derive",
+ "crossterm",
  "csv",
  "env_logger",
  "itertools",
@@ -682,6 +760,15 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "regex"
 version = "1.10.5"
@@ -739,6 +826,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "semver"
 version = "1.0.23"
@@ -809,6 +902,42 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "signal-hook"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-mio"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
+dependencies = [
+ "libc",
+ "mio",
+ "signal-hook",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -906,6 +1035,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.92"
@@ -960,6 +1095,22 @@ version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
 
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
 [[package]]
 name = "winapi-util"
 version = "0.1.8"
@@ -969,6 +1120,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-core"
 version = "0.52.0"

Cargo.toml

@@ -17,6 +17,7 @@ regex = "1.10"
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "3.8"
 csv = "1.3"
+crossterm = "0.28"
 
 [dev-dependencies]
 assert_cmd = "2.0"

src/main.rs

@@ -1,15 +1,17 @@
 use core::str;
 use std::{
+    env,
     fmt::Display,
-    fs::File,
+    fs::{self, File},
-    io::{BufRead, BufReader},
+    io::{self, BufRead, BufReader},
-    path::{self, PathBuf},
+    path::{self, Path, PathBuf},
     str::FromStr,
 };
 
 use anyhow::{anyhow, Context, Result};
 use chrono::{DateTime, Utc};
 use clap::{Parser, Subcommand};
+use crossterm::{event, terminal};
 use log::{debug, error, info, trace, warn};
 use masterlog::{get_output_filename, MasterLogResult};
 use semver::Version;
@@ -31,7 +33,7 @@ struct Args {
 
 #[derive(Subcommand, Debug)]
 pub enum Commands {
-    /// Parse master jathub logfile for PS Board QAQC and write out to CSV.
+    /// Parse master log and convert to CSV and prompt post processes interactively.
     AddMasterLog {
         /// Master log file.
         master_log: path::PathBuf,
@@ -39,6 +41,19 @@ pub enum Commands {
         /// Default is `out_<runid>.csv`
         // #[arg(default_value = get_default_log_path().into_os_string())]
         outfile: Option<PathBuf>,
+        /// Editor to use.
+        /// Default is `$VISUAL` or `$EDITOR`.
+        #[arg(short, long)]
+        editor: Option<String>,
+    },
+    /// Parse master jathub logfile for PS Board QAQC and write out to CSV.
+    ConvertMasterLog {
+        /// Master log file.
+        master_log: path::PathBuf,
+        /// Output CSV file.
+        /// Default is `out_<runid>.csv`
+        // #[arg(default_value = get_default_log_path().into_os_string())]
+        outfile: Option<PathBuf>,
     },
     /// Check CSV format
     CheckDB {
@@ -336,6 +351,38 @@ impl PsbQaqcResult {
 //         .collect_vec()
 // }
 
+fn write_psbqaqc_csv(result: MasterLogResult, outfile: PathBuf) -> Result<()> {
+    let expanded_results = PsbQaqcResult::from_masterlogresult(result);
+
+    let mut wtr = match outfile.exists() {
+        true => {
+            let file = File::options()
+                .read(true)
+                .append(true)
+                .open(outfile.clone())?;
+            csv::WriterBuilder::new()
+                .has_headers(false)
+                .from_writer(file)
+        }
+        false => {
+            println!("Creating new file: {}", outfile.display());
+            let file = File::options()
+                .create_new(true)
+                .write(true)
+                .open(outfile.clone())?;
+            csv::WriterBuilder::new()
+                .has_headers(true)
+                .from_writer(file)
+        }
+    };
+    for result in expanded_results {
+        wtr.serialize(result)?;
+    }
+    wtr.flush()?;
+
+    Ok(())
+}
+
 fn main() -> Result<()> {
     let args = Args::parse();
     env_logger::Builder::new()
@@ -352,55 +399,91 @@ fn main() -> Result<()> {
         Commands::AddMasterLog {
             master_log,
             outfile,
+            editor,
         } => {
-            let result = {
+            let result = MasterLogResult::parse_file(master_log)?;
-                let file = File::open(master_log.clone())?;
-                let reader = BufReader::new(file);
-                MasterLogResult::parse_file(
-                    reader,
-                    master_log
-                        .file_name()
-                        .unwrap()
-                        .to_str()
-                        .unwrap()
-                        .to_string(),
-                )?
-            };
             debug!("{:?}", result);
 
             // Print boards to retest
-            for (pos, boardresult) in &result.board_results {
+            result.print_boards_to_retest(io::stdout())?;
-                if let Some(reason) = boardresult.is_need_retest() {
-                    println!(
-                        "Board {} at {} need retest for {}",
-                        boardresult.id.id, pos, reason
-                    );
-                }
-            }
 
             let outfile = outfile.unwrap_or(get_output_filename(&result));
 
-            let expanded_results = PsbQaqcResult::from_masterlogresult(result);
+            write_psbqaqc_csv(result, outfile.clone())?;
 
-            let mut wtr = match outfile.exists() {
+            println!("Add comments about errors in the last column of the CSV file");
-                true => {
+            println!("Press any key to start editting...");
-                    let file = File::options().read(true).append(true).open(outfile)?;
+            terminal::enable_raw_mode()?;
-                    csv::WriterBuilder::new()
+            let _ = event::read()?;
-                        .has_headers(false)
+            terminal::disable_raw_mode()?;
-                        .from_writer(file)
+
+            let editor = match (editor, env::var("VISUAL"), env::var("EDITOR")) {
+                (Some(editor), _, _) => editor,
+                (None, Ok(editor), _) => {
+                    info!("Use VISUAL");
+                    editor
                 }
-                false => {
+                (None, Err(_), Ok(editor)) => {
-                    println!("Creating new file: {}", outfile.display());
+                    info!("Use EDITOR");
-                    let file = File::options().create_new(true).write(true).open(outfile)?;
+                    editor
-                    csv::WriterBuilder::new()
+                }
-                        .has_headers(true)
+                (None, Err(e1), Err(e2)) => {
-                        .from_writer(file)
+                    info!("No VISUAL nor EDITOR found, {}, {}", e1, e2);
+                    "nano".to_string()
                 }
             };
-            for result in expanded_results {
+            std::process::Command::new(editor)
-                wtr.serialize(result)?;
+                .arg(outfile.clone())
+                .spawn()?
+                .wait()?;
+
+            {
+                let f = File::open(outfile.clone())?;
+                let rdr = BufReader::new(f);
+                rdr.lines().for_each(|l| println!("{}", l.unwrap()));
             }
-            wtr.flush()?;
+            println!();
+            println!("Copy the CSV above and paste it to the database(Google sheets)");
+            println!("Choose Data->Split text to columns to format it");
+
+            println!("Press any key when upload finished...");
+            terminal::enable_raw_mode()?;
+            let _ = event::read()?;
+            terminal::disable_raw_mode()?;
+
+            let uploaded_outfile = {
+                let mut new_file_name = outfile
+                    .clone()
+                    .file_stem()
+                    .context("No file stem for out file")?
+                    .to_owned();
+                new_file_name.push("_uploaded");
+                if let Some(ext) = outfile.extension() {
+                    new_file_name.push(".");
+                    new_file_name.push(ext);
+                };
+                PathBuf::from("log").join(new_file_name)
+            };
+            info!(
+                "Renaming {} to {}",
+                outfile.display(),
+                uploaded_outfile.is_dir()
+            );
+            fs::rename(outfile, uploaded_outfile)?;
+        }
+        Commands::ConvertMasterLog {
+            master_log,
+            outfile,
+        } => {
+            let result = MasterLogResult::parse_file(master_log)?;
+            debug!("{:?}", result);
+
+            // Print boards to retest
+            result.print_boards_to_retest(io::stdout())?;
+
+            let outfile = outfile.unwrap_or(get_output_filename(&result));
+
+            write_psbqaqc_csv(result, outfile)?;
         }
         Commands::CheckDB { csvfile } => {
             // TODO: more friendly message (like row number)

src/masterlog.rs

@@ -1,4 +1,10 @@
-use std::{collections::BTreeMap, io::BufRead, path::PathBuf, str::FromStr};
+use std::{
+    collections::BTreeMap,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    path::PathBuf,
+    str::FromStr,
+};
 
 use anyhow::{anyhow, Context, Result};
 use chrono::{DateTime, Utc};
@@ -102,7 +108,21 @@ fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
 
 impl MasterLogResult {
     /// Parse log file on master jathub.
-    pub fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
+    pub fn parse_file(master_log: PathBuf) -> Result<MasterLogResult> {
+        let file = File::open(master_log.clone())?;
+        let reader = BufReader::new(file);
+        MasterLogResult::parse_file_with_filename(
+            reader,
+            master_log
+                .file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .to_string(),
+        )
+    }
+
+    fn parse_file_with_filename(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
         let mut lines = file.lines();
 
         let version = {
@@ -301,6 +321,19 @@ impl MasterLogResult {
             filename,
         })
     }
+
+    pub fn print_boards_to_retest(&self, mut wtr: impl Write) -> Result<()> {
+        for (pos, boardresult) in &self.board_results {
+            if let Some(reason) = boardresult.is_need_retest() {
+                writeln!(
+                    wtr,
+                    "Board {} at {} need retest for {}",
+                    boardresult.id.id, pos, reason
+                )?;
+            }
+        }
+        Ok(())
+    }
 }
 
 pub fn get_output_filename(result: &MasterLogResult) -> PathBuf {

tests/cli.rs

@@ -20,7 +20,7 @@ mod integrated_test {
         // 1st file
         let mut cmd = Command::cargo_bin("psb-qaqc")?;
         cmd.current_dir("tests")
-            .arg("add-master-log")
+            .arg("convert-master-log")
             .arg("./example_logs/valid/44.log")
             .arg(test_out.as_path())
             .assert()
@@ -49,7 +49,7 @@ mod integrated_test {
         // 2nd file
         let mut cmd = Command::cargo_bin("psb-qaqc")?;
         cmd.current_dir("tests")
-            .arg("add-master-log")
+            .arg("convert-master-log")
             .arg("./example_logs/valid/20.log")
             .arg(test_out.as_path())
             .assert()
@@ -77,7 +77,7 @@ mod integrated_test {
 
         let mut cmd = Command::cargo_bin("psb-qaqc")?;
         cmd.current_dir(&test_out_dir)
-            .arg("add-master-log")
+            .arg("convert-master-log")
             .arg("84.log")
             .assert()
             .success()