mirror of
https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess.git
synced 2024-11-22 07:10:18 +09:00
Merge branch 'feature-more-postprocess' into 'main'
[NEW] add more postprocesses to `add-master-log` and rename current one as `convert-master-log` See merge request wotsubo/psboard-qaqc-postprocess!1
This commit is contained in:
commit
a4c0ac9c55
6 changed files with 328 additions and 46 deletions
|
|
@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- `add-master-log` subcommand provide interactive session to upload master log to database via CSV
|
||||
|
||||
### Change
|
||||
|
||||
- *BREAKING* old `add-master-log` is now renamed to `convert-master-log`
|
||||
|
||||
## [1.1.0]
|
||||
|
||||
### Added
|
||||
|
|
|
|||
157
Cargo.lock
generated
157
Cargo.lock
generated
|
|
@ -260,6 +260,31 @@ version = "0.8.20"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
|
||||
|
||||
[[package]]
|
||||
name = "crossterm"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crossterm_winapi",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"rustix",
|
||||
"signal-hook",
|
||||
"signal-hook-mio",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm_winapi"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv"
|
||||
version = "1.3.0"
|
||||
|
|
@ -446,6 +471,12 @@ version = "0.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
|
|
@ -567,6 +598,16 @@ version = "0.4.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.22"
|
||||
|
|
@ -579,6 +620,19 @@ version = "2.7.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"log",
|
||||
"wasi",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "normalize-line-endings"
|
||||
version = "0.3.0"
|
||||
|
|
@ -606,6 +660,29 @@ version = "1.19.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
|
|
@ -662,6 +739,7 @@ dependencies = [
|
|||
"clap",
|
||||
"clap-verbosity-flag",
|
||||
"clap_derive",
|
||||
"crossterm",
|
||||
"csv",
|
||||
"env_logger",
|
||||
"itertools",
|
||||
|
|
@ -682,6 +760,15 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.10.5"
|
||||
|
|
@ -739,6 +826,12 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.23"
|
||||
|
|
@ -809,6 +902,42 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"signal-hook-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-mio"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"mio",
|
||||
"signal-hook",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
|
|
@ -906,6 +1035,12 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.92"
|
||||
|
|
@ -960,6 +1095,22 @@ version = "0.2.92"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.8"
|
||||
|
|
@ -969,6 +1120,12 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.52.0"
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ regex = "1.10"
|
|||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_with = "3.8"
|
||||
csv = "1.3"
|
||||
crossterm = "0.28"
|
||||
|
||||
[dev-dependencies]
|
||||
assert_cmd = "2.0"
|
||||
|
|
|
|||
165
src/main.rs
165
src/main.rs
|
|
@ -1,15 +1,17 @@
|
|||
use core::str;
|
||||
use std::{
|
||||
env,
|
||||
fmt::Display,
|
||||
fs::File,
|
||||
io::{BufRead, BufReader},
|
||||
path::{self, PathBuf},
|
||||
fs::{self, File},
|
||||
io::{self, BufRead, BufReader},
|
||||
path::{self, Path, PathBuf},
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use chrono::{DateTime, Utc};
|
||||
use clap::{Parser, Subcommand};
|
||||
use crossterm::{event, terminal};
|
||||
use log::{debug, error, info, trace, warn};
|
||||
use masterlog::{get_output_filename, MasterLogResult};
|
||||
use semver::Version;
|
||||
|
|
@ -31,7 +33,7 @@ struct Args {
|
|||
|
||||
#[derive(Subcommand, Debug)]
|
||||
pub enum Commands {
|
||||
/// Parse master jathub logfile for PS Board QAQC and write out to CSV.
|
||||
/// Parse master log and convert to CSV and prompt post processes interactively.
|
||||
AddMasterLog {
|
||||
/// Master log file.
|
||||
master_log: path::PathBuf,
|
||||
|
|
@ -39,6 +41,19 @@ pub enum Commands {
|
|||
/// Default is `out_<runid>.csv`
|
||||
// #[arg(default_value = get_default_log_path().into_os_string())]
|
||||
outfile: Option<PathBuf>,
|
||||
/// Editor to use.
|
||||
/// Default is `$VISUAL` or `$EDITOR`.
|
||||
#[arg(short, long)]
|
||||
editor: Option<String>,
|
||||
},
|
||||
/// Parse master jathub logfile for PS Board QAQC and write out to CSV.
|
||||
ConvertMasterLog {
|
||||
/// Master log file.
|
||||
master_log: path::PathBuf,
|
||||
/// Output CSV file.
|
||||
/// Default is `out_<runid>.csv`
|
||||
// #[arg(default_value = get_default_log_path().into_os_string())]
|
||||
outfile: Option<PathBuf>,
|
||||
},
|
||||
/// Check CSV format
|
||||
CheckDB {
|
||||
|
|
@ -336,6 +351,38 @@ impl PsbQaqcResult {
|
|||
// .collect_vec()
|
||||
// }
|
||||
|
||||
fn write_psbqaqc_csv(result: MasterLogResult, outfile: PathBuf) -> Result<()> {
|
||||
let expanded_results = PsbQaqcResult::from_masterlogresult(result);
|
||||
|
||||
let mut wtr = match outfile.exists() {
|
||||
true => {
|
||||
let file = File::options()
|
||||
.read(true)
|
||||
.append(true)
|
||||
.open(outfile.clone())?;
|
||||
csv::WriterBuilder::new()
|
||||
.has_headers(false)
|
||||
.from_writer(file)
|
||||
}
|
||||
false => {
|
||||
println!("Creating new file: {}", outfile.display());
|
||||
let file = File::options()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(outfile.clone())?;
|
||||
csv::WriterBuilder::new()
|
||||
.has_headers(true)
|
||||
.from_writer(file)
|
||||
}
|
||||
};
|
||||
for result in expanded_results {
|
||||
wtr.serialize(result)?;
|
||||
}
|
||||
wtr.flush()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
env_logger::Builder::new()
|
||||
|
|
@ -352,55 +399,91 @@ fn main() -> Result<()> {
|
|||
Commands::AddMasterLog {
|
||||
master_log,
|
||||
outfile,
|
||||
editor,
|
||||
} => {
|
||||
let result = {
|
||||
let file = File::open(master_log.clone())?;
|
||||
let reader = BufReader::new(file);
|
||||
MasterLogResult::parse_file(
|
||||
reader,
|
||||
master_log
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
)?
|
||||
};
|
||||
let result = MasterLogResult::parse_file(master_log)?;
|
||||
debug!("{:?}", result);
|
||||
|
||||
// Print boards to retest
|
||||
for (pos, boardresult) in &result.board_results {
|
||||
if let Some(reason) = boardresult.is_need_retest() {
|
||||
println!(
|
||||
"Board {} at {} need retest for {}",
|
||||
boardresult.id.id, pos, reason
|
||||
);
|
||||
}
|
||||
}
|
||||
result.print_boards_to_retest(io::stdout())?;
|
||||
|
||||
let outfile = outfile.unwrap_or(get_output_filename(&result));
|
||||
|
||||
let expanded_results = PsbQaqcResult::from_masterlogresult(result);
|
||||
write_psbqaqc_csv(result, outfile.clone())?;
|
||||
|
||||
let mut wtr = match outfile.exists() {
|
||||
true => {
|
||||
let file = File::options().read(true).append(true).open(outfile)?;
|
||||
csv::WriterBuilder::new()
|
||||
.has_headers(false)
|
||||
.from_writer(file)
|
||||
println!("Add comments about errors in the last column of the CSV file");
|
||||
println!("Press any key to start editting...");
|
||||
terminal::enable_raw_mode()?;
|
||||
let _ = event::read()?;
|
||||
terminal::disable_raw_mode()?;
|
||||
|
||||
let editor = match (editor, env::var("VISUAL"), env::var("EDITOR")) {
|
||||
(Some(editor), _, _) => editor,
|
||||
(None, Ok(editor), _) => {
|
||||
info!("Use VISUAL");
|
||||
editor
|
||||
}
|
||||
false => {
|
||||
println!("Creating new file: {}", outfile.display());
|
||||
let file = File::options().create_new(true).write(true).open(outfile)?;
|
||||
csv::WriterBuilder::new()
|
||||
.has_headers(true)
|
||||
.from_writer(file)
|
||||
(None, Err(_), Ok(editor)) => {
|
||||
info!("Use EDITOR");
|
||||
editor
|
||||
}
|
||||
(None, Err(e1), Err(e2)) => {
|
||||
info!("No VISUAL nor EDITOR found, {}, {}", e1, e2);
|
||||
"nano".to_string()
|
||||
}
|
||||
};
|
||||
for result in expanded_results {
|
||||
wtr.serialize(result)?;
|
||||
std::process::Command::new(editor)
|
||||
.arg(outfile.clone())
|
||||
.spawn()?
|
||||
.wait()?;
|
||||
|
||||
{
|
||||
let f = File::open(outfile.clone())?;
|
||||
let rdr = BufReader::new(f);
|
||||
rdr.lines().for_each(|l| println!("{}", l.unwrap()));
|
||||
}
|
||||
wtr.flush()?;
|
||||
println!();
|
||||
println!("Copy the CSV above and paste it to the database(Google sheets)");
|
||||
println!("Choose Data->Split text to columns to format it");
|
||||
|
||||
println!("Press any key when upload finished...");
|
||||
terminal::enable_raw_mode()?;
|
||||
let _ = event::read()?;
|
||||
terminal::disable_raw_mode()?;
|
||||
|
||||
let uploaded_outfile = {
|
||||
let mut new_file_name = outfile
|
||||
.clone()
|
||||
.file_stem()
|
||||
.context("No file stem for out file")?
|
||||
.to_owned();
|
||||
new_file_name.push("_uploaded");
|
||||
if let Some(ext) = outfile.extension() {
|
||||
new_file_name.push(".");
|
||||
new_file_name.push(ext);
|
||||
};
|
||||
PathBuf::from("log").join(new_file_name)
|
||||
};
|
||||
info!(
|
||||
"Renaming {} to {}",
|
||||
outfile.display(),
|
||||
uploaded_outfile.is_dir()
|
||||
);
|
||||
fs::rename(outfile, uploaded_outfile)?;
|
||||
}
|
||||
Commands::ConvertMasterLog {
|
||||
master_log,
|
||||
outfile,
|
||||
} => {
|
||||
let result = MasterLogResult::parse_file(master_log)?;
|
||||
debug!("{:?}", result);
|
||||
|
||||
// Print boards to retest
|
||||
result.print_boards_to_retest(io::stdout())?;
|
||||
|
||||
let outfile = outfile.unwrap_or(get_output_filename(&result));
|
||||
|
||||
write_psbqaqc_csv(result, outfile)?;
|
||||
}
|
||||
Commands::CheckDB { csvfile } => {
|
||||
// TODO: more friendly message (like row number)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
use std::{collections::BTreeMap, io::BufRead, path::PathBuf, str::FromStr};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs::File,
|
||||
io::{BufRead, BufReader, Write},
|
||||
path::PathBuf,
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use chrono::{DateTime, Utc};
|
||||
|
|
@ -102,7 +108,21 @@ fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
|
|||
|
||||
impl MasterLogResult {
|
||||
/// Parse log file on master jathub.
|
||||
pub fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
|
||||
pub fn parse_file(master_log: PathBuf) -> Result<MasterLogResult> {
|
||||
let file = File::open(master_log.clone())?;
|
||||
let reader = BufReader::new(file);
|
||||
MasterLogResult::parse_file_with_filename(
|
||||
reader,
|
||||
master_log
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_file_with_filename(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
|
||||
let mut lines = file.lines();
|
||||
|
||||
let version = {
|
||||
|
|
@ -301,6 +321,19 @@ impl MasterLogResult {
|
|||
filename,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn print_boards_to_retest(&self, mut wtr: impl Write) -> Result<()> {
|
||||
for (pos, boardresult) in &self.board_results {
|
||||
if let Some(reason) = boardresult.is_need_retest() {
|
||||
writeln!(
|
||||
wtr,
|
||||
"Board {} at {} need retest for {}",
|
||||
boardresult.id.id, pos, reason
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_output_filename(result: &MasterLogResult) -> PathBuf {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ mod integrated_test {
|
|||
// 1st file
|
||||
let mut cmd = Command::cargo_bin("psb-qaqc")?;
|
||||
cmd.current_dir("tests")
|
||||
.arg("add-master-log")
|
||||
.arg("convert-master-log")
|
||||
.arg("./example_logs/valid/44.log")
|
||||
.arg(test_out.as_path())
|
||||
.assert()
|
||||
|
|
@ -49,7 +49,7 @@ mod integrated_test {
|
|||
// 2nd file
|
||||
let mut cmd = Command::cargo_bin("psb-qaqc")?;
|
||||
cmd.current_dir("tests")
|
||||
.arg("add-master-log")
|
||||
.arg("convert-master-log")
|
||||
.arg("./example_logs/valid/20.log")
|
||||
.arg(test_out.as_path())
|
||||
.assert()
|
||||
|
|
@ -77,7 +77,7 @@ mod integrated_test {
|
|||
|
||||
let mut cmd = Command::cargo_bin("psb-qaqc")?;
|
||||
cmd.current_dir(&test_out_dir)
|
||||
.arg("add-master-log")
|
||||
.arg("convert-master-log")
|
||||
.arg("84.log")
|
||||
.assert()
|
||||
.success()
|
||||
|
|
|
|||
Loading…
Reference in a new issue