Compare commits

...

12 commits
v1.1.0 ... main

Author SHA1 Message Date
a78416a18d fix: CHANGELOG diff links 2024-09-11 15:31:09 +09:00
2dd0c20ce4 update CHANGELOG 2024-09-11 15:25:50 +09:00
3fdbe5ad4c update: print error when no editor is specified & add-master-log doesn't
append csv file
2024-09-11 15:19:13 +09:00
4478ec1654 bump to v2.0.1
fix version in  Cargo.toml
2024-09-11 10:31:28 +09:00
6842097962 bump to v2.0.0 2024-09-11 10:17:49 +09:00
c8525b2ec6 Add LICENSE 2024-09-11 03:04:52 +02:00
a4c0ac9c55 Merge branch 'feature-more-postprocess' into 'main'
[NEW] add more postprocesses to `add-master-log` and rename current one as `convert-master-log`

See merge request wotsubo/psboard-qaqc-postprocess!1
2024-09-11 03:01:51 +02:00
f9716bcc93 update CHANGELOG for add-master-log 2024-09-10 23:04:48 +09:00
2dc790f9a8 new: add interactive command to add master log to database
- old add-master-log is now convert-master-log
- add-master-log is now interactive
- add-master-log is not well tested
2024-09-10 23:01:20 +09:00
7fe5218d39 refactor: split csv writing to func 2024-09-10 21:52:41 +09:00
7529d003a6 refactor: replace MasterLogResult file parse API to use only filename
* old one moved to private and used in the new one
* for future convert-master-log replacement
2024-09-10 21:22:25 +09:00
9e93a085fd refactor: split func for printing boards for retest 2024-09-10 21:07:50 +09:00
7 changed files with 376 additions and 50 deletions

View file

@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- `add-master-log` now always create CSV file
- `convert-master-log` can still append to a CSV file
## [2.0.1]
### Fix
- version in Cargo.toml
## [2.0.0]
### Added
- `add-master-log` subcommand provide interactive session to upload master log to database via CSV
### Change
- *BREAKING* old `add-master-log` is now renamed to `convert-master-log`
## [1.1.0]
### Added
@ -33,7 +54,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Subcommand `add-master-log` to parse master log for shiftwork 0.1.0 and write out to CSV.
- Subcommand `check-db` to validate the database CSV file.
[Unreleased]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/main...v1.0.1
[1.1.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.1.0...v1.0.1
[1.0.1]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.0.1...v1.0.0
[Unreleased]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.0.1...main
[2.0.1]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v2.0.0...v2.0.1
[2.0.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.1.0...v2.0.0
[1.1.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.0.1...v1.1.0
[1.0.1]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/compare/v1.0.0...v1.0.1
[1.0.0]: https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess/-/tags/v1.0.0

159
Cargo.lock generated
View file

@ -260,6 +260,31 @@ version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "crossterm"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
dependencies = [
"bitflags",
"crossterm_winapi",
"mio",
"parking_lot",
"rustix",
"signal-hook",
"signal-hook-mio",
"winapi",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
dependencies = [
"winapi",
]
[[package]]
name = "csv"
version = "1.3.0"
@ -446,6 +471,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
[[package]]
name = "hex"
version = "0.4.3"
@ -567,6 +598,16 @@ version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
[[package]]
name = "lock_api"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.22"
@ -579,6 +620,19 @@ version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "mio"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
dependencies = [
"hermit-abi",
"libc",
"log",
"wasi",
"windows-sys",
]
[[package]]
name = "normalize-line-endings"
version = "0.3.0"
@ -606,6 +660,29 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "parking_lot"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
name = "powerfmt"
version = "0.2.0"
@ -653,7 +730,7 @@ dependencies = [
[[package]]
name = "psb-qaqc"
version = "1.1.0"
version = "2.0.1"
dependencies = [
"anyhow",
"assert_cmd",
@ -662,6 +739,7 @@ dependencies = [
"clap",
"clap-verbosity-flag",
"clap_derive",
"crossterm",
"csv",
"env_logger",
"itertools",
@ -682,6 +760,15 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.10.5"
@ -739,6 +826,12 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
version = "1.0.23"
@ -809,6 +902,42 @@ dependencies = [
"syn",
]
[[package]]
name = "signal-hook"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
dependencies = [
"libc",
"signal-hook-registry",
]
[[package]]
name = "signal-hook-mio"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
dependencies = [
"libc",
"mio",
"signal-hook",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
dependencies = [
"libc",
]
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "strsim"
version = "0.11.1"
@ -906,6 +1035,12 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
@ -960,6 +1095,22 @@ version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.8"
@ -969,6 +1120,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.52.0"

View file

@ -1,6 +1,6 @@
[package]
name = "psb-qaqc"
version = "1.1.0"
version = "2.0.1"
edition = "2021"
[dependencies]
@ -17,6 +17,7 @@ regex = "1.10"
serde = { version = "1.0", features = ["derive"] }
serde_with = "3.8"
csv = "1.3"
crossterm = "0.28"
[dev-dependencies]
assert_cmd = "2.0"

21
LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 Wataru Otsubo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View file

@ -1,8 +1,9 @@
use core::str;
use std::{
env,
fmt::Display,
fs::File,
io::{BufRead, BufReader},
fs::{self, File},
io::{self, BufRead, BufReader},
path::{self, PathBuf},
str::FromStr,
};
@ -10,6 +11,7 @@ use std::{
use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Utc};
use clap::{Parser, Subcommand};
use crossterm::{event, terminal};
use log::{debug, error, info, trace, warn};
use masterlog::{get_output_filename, MasterLogResult};
use semver::Version;
@ -31,7 +33,7 @@ struct Args {
#[derive(Subcommand, Debug)]
pub enum Commands {
/// Parse master jathub logfile for PS Board QAQC and write out to CSV.
/// Parse master log and convert to CSV and prompt post processes interactively.
AddMasterLog {
/// Master log file.
master_log: path::PathBuf,
@ -39,6 +41,19 @@ pub enum Commands {
/// Default is `out_<runid>.csv`
// #[arg(default_value = get_default_log_path().into_os_string())]
outfile: Option<PathBuf>,
/// Editor to use.
/// Default is `$VISUAL` or `$EDITOR`.
#[arg(short, long)]
editor: Option<String>,
},
/// Parse master jathub logfile for PS Board QAQC and write out to CSV.
ConvertMasterLog {
/// Master log file.
master_log: path::PathBuf,
/// Output CSV file.
/// Default is `out_<runid>.csv`
// #[arg(default_value = get_default_log_path().into_os_string())]
outfile: Option<PathBuf>,
},
/// Check CSV format
CheckDB {
@ -336,6 +351,43 @@ impl PsbQaqcResult {
// .collect_vec()
// }
/// If `nonewfile` is `true`, out file is not created when there is already a file.
fn write_psbqaqc_csv(result: MasterLogResult, outfile: PathBuf, nonewfile: bool) -> Result<()> {
let expanded_results = PsbQaqcResult::from_masterlogresult(result);
let mut wtr = match (outfile.exists(), nonewfile) {
(true, false) => {
let file = File::options()
.read(true)
.append(true)
.open(outfile.clone())?;
csv::WriterBuilder::new()
.has_headers(false)
.from_writer(file)
}
(false, _) => {
println!("Creating new file: {}", outfile.display());
let file = File::options()
.create_new(true)
.write(true)
.open(outfile.clone())?;
csv::WriterBuilder::new()
.has_headers(true)
.from_writer(file)
}
(true, true) => {
error!("Out file already exists but specified not to make new file");
return Err(anyhow!("Out file already exists"));
}
};
for result in expanded_results {
wtr.serialize(result)?;
}
wtr.flush()?;
Ok(())
}
fn main() -> Result<()> {
let args = Args::parse();
env_logger::Builder::new()
@ -352,55 +404,94 @@ fn main() -> Result<()> {
Commands::AddMasterLog {
master_log,
outfile,
editor,
} => {
let result = {
let file = File::open(master_log.clone())?;
let reader = BufReader::new(file);
MasterLogResult::parse_file(
reader,
master_log
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_string(),
)?
};
let result = MasterLogResult::parse_file(master_log)?;
debug!("{:?}", result);
// Print boards to retest
for (pos, boardresult) in &result.board_results {
if let Some(reason) = boardresult.is_need_retest() {
println!(
"Board {} at {} need retest for {}",
boardresult.id.id, pos, reason
);
}
}
result.print_boards_to_retest(io::stdout())?;
let outfile = outfile.unwrap_or(get_output_filename(&result));
let expanded_results = PsbQaqcResult::from_masterlogresult(result);
write_psbqaqc_csv(result, outfile.clone(), true)?;
let mut wtr = match outfile.exists() {
true => {
let file = File::options().read(true).append(true).open(outfile)?;
csv::WriterBuilder::new()
.has_headers(false)
.from_writer(file)
println!("Add comments about errors in the last column of the CSV file");
println!("Press any key to start editting...");
terminal::enable_raw_mode()?;
let _ = event::read()?;
terminal::disable_raw_mode()?;
let editor = match (editor, env::var("VISUAL"), env::var("EDITOR")) {
(Some(editor), _, _) => editor,
(None, Ok(editor), _) => {
info!("Use VISUAL");
editor
}
false => {
println!("Creating new file: {}", outfile.display());
let file = File::options().create_new(true).write(true).open(outfile)?;
csv::WriterBuilder::new()
.has_headers(true)
.from_writer(file)
(None, Err(_), Ok(editor)) => {
info!("Use EDITOR");
editor
}
(None, Err(e1), Err(e2)) => {
warn!("No VISUAL nor EDITOR found, {}, {}", e1, e2);
warn!("Using `nano` as a fallback editor");
"nano".to_string()
}
};
for result in expanded_results {
wtr.serialize(result)?;
std::process::Command::new(editor)
.arg(outfile.clone())
.spawn()
.context("Error during spawning editor")?
.wait()
.context("Error while waiting editor")?;
{
let f = File::open(outfile.clone())?;
let rdr = BufReader::new(f);
rdr.lines().for_each(|l| println!("{}", l.unwrap()));
}
wtr.flush()?;
println!();
println!("Copy the CSV above and paste it to the database(Google sheets)");
println!("Choose `Data->Split text to columns` to format it");
println!("Press any key when upload finished...");
terminal::enable_raw_mode()?;
let _ = event::read()?;
terminal::disable_raw_mode()?;
let uploaded_outfile = {
let mut new_file_name = outfile
.clone()
.file_stem()
.context("No file stem for out file")?
.to_owned();
new_file_name.push("_uploaded");
if let Some(ext) = outfile.extension() {
new_file_name.push(".");
new_file_name.push(ext);
};
PathBuf::from("log").join(new_file_name)
};
info!(
"Renaming {} to {}",
outfile.display(),
uploaded_outfile.is_dir()
);
fs::rename(outfile, uploaded_outfile)?;
}
Commands::ConvertMasterLog {
master_log,
outfile,
} => {
let result = MasterLogResult::parse_file(master_log)?;
debug!("{:?}", result);
// Print boards to retest
result.print_boards_to_retest(io::stdout())?;
let outfile = outfile.unwrap_or(get_output_filename(&result));
write_psbqaqc_csv(result, outfile, false)?;
}
Commands::CheckDB { csvfile } => {
// TODO: more friendly message (like row number)

View file

@ -1,4 +1,10 @@
use std::{collections::BTreeMap, io::BufRead, path::PathBuf, str::FromStr};
use std::{
collections::BTreeMap,
fs::File,
io::{BufRead, BufReader, Write},
path::PathBuf,
str::FromStr,
};
use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Utc};
@ -102,7 +108,21 @@ fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
impl MasterLogResult {
/// Parse log file on master jathub.
pub fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
pub fn parse_file(master_log: PathBuf) -> Result<MasterLogResult> {
let file = File::open(master_log.clone())?;
let reader = BufReader::new(file);
MasterLogResult::parse_file_with_filename(
reader,
master_log
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_string(),
)
}
fn parse_file_with_filename(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
let mut lines = file.lines();
let version = {
@ -301,6 +321,19 @@ impl MasterLogResult {
filename,
})
}
pub fn print_boards_to_retest(&self, mut wtr: impl Write) -> Result<()> {
for (pos, boardresult) in &self.board_results {
if let Some(reason) = boardresult.is_need_retest() {
writeln!(
wtr,
"Board {} at {} need retest for {}",
boardresult.id.id, pos, reason
)?;
}
}
Ok(())
}
}
pub fn get_output_filename(result: &MasterLogResult) -> PathBuf {

View file

@ -20,7 +20,7 @@ mod integrated_test {
// 1st file
let mut cmd = Command::cargo_bin("psb-qaqc")?;
cmd.current_dir("tests")
.arg("add-master-log")
.arg("convert-master-log")
.arg("./example_logs/valid/44.log")
.arg(test_out.as_path())
.assert()
@ -49,7 +49,7 @@ mod integrated_test {
// 2nd file
let mut cmd = Command::cargo_bin("psb-qaqc")?;
cmd.current_dir("tests")
.arg("add-master-log")
.arg("convert-master-log")
.arg("./example_logs/valid/20.log")
.arg(test_out.as_path())
.assert()
@ -77,7 +77,7 @@ mod integrated_test {
let mut cmd = Command::cargo_bin("psb-qaqc")?;
cmd.current_dir(&test_out_dir)
.arg("add-master-log")
.arg("convert-master-log")
.arg("84.log")
.assert()
.success()