mirror of
https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess.git
synced 2025-04-20 19:56:06 +09:00
Wataru Otsubo
2dc790f9a8
- old add-master-log is now convert-master-log - add-master-log is now interactive - add-master-log is not well tested
661 lines
20 KiB
Rust
661 lines
20 KiB
Rust
use core::str;
|
|
use std::{
|
|
env,
|
|
fmt::Display,
|
|
fs::{self, File},
|
|
io::{self, BufRead, BufReader},
|
|
path::{self, Path, PathBuf},
|
|
str::FromStr,
|
|
};
|
|
|
|
use anyhow::{anyhow, Context, Result};
|
|
use chrono::{DateTime, Utc};
|
|
use clap::{Parser, Subcommand};
|
|
use crossterm::{event, terminal};
|
|
use log::{debug, error, info, trace, warn};
|
|
use masterlog::{get_output_filename, MasterLogResult};
|
|
use semver::Version;
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_with::{serde_as, DisplayFromStr};
|
|
|
|
mod masterlog;
|
|
mod skew;
|
|
|
|
/// PS Board QAQC shift related commands.
|
|
#[derive(Parser, Debug)]
|
|
#[command(version, about, long_about = None)]
|
|
struct Args {
|
|
#[command(subcommand)]
|
|
command: Commands,
|
|
#[command(flatten)]
|
|
verbose: clap_verbosity_flag::Verbosity,
|
|
}
|
|
|
|
#[derive(Subcommand, Debug)]
|
|
pub enum Commands {
|
|
/// Parse master log and convert to CSV and prompt post processes interactively.
|
|
AddMasterLog {
|
|
/// Master log file.
|
|
master_log: path::PathBuf,
|
|
/// Output CSV file.
|
|
/// Default is `out_<runid>.csv`
|
|
// #[arg(default_value = get_default_log_path().into_os_string())]
|
|
outfile: Option<PathBuf>,
|
|
/// Editor to use.
|
|
/// Default is `$VISUAL` or `$EDITOR`.
|
|
#[arg(short, long)]
|
|
editor: Option<String>,
|
|
},
|
|
/// Parse master jathub logfile for PS Board QAQC and write out to CSV.
|
|
ConvertMasterLog {
|
|
/// Master log file.
|
|
master_log: path::PathBuf,
|
|
/// Output CSV file.
|
|
/// Default is `out_<runid>.csv`
|
|
// #[arg(default_value = get_default_log_path().into_os_string())]
|
|
outfile: Option<PathBuf>,
|
|
},
|
|
/// Check CSV format
|
|
CheckDB {
|
|
/// Database CSV file.
|
|
csvfile: PathBuf,
|
|
},
|
|
/// Get LVDS TX skew from slave log _clk file.
|
|
///
|
|
/// PSB ID is extracted from `logfile` name and the corresponding QAQC run is retrieved by
|
|
/// searching the n-th run with the same PSB ID in chronological order.
|
|
AddSkew {
|
|
/// Slave _clk log file.
|
|
logfile: PathBuf,
|
|
/// Output CSV file.
|
|
outfile: PathBuf,
|
|
},
|
|
}
|
|
|
|
/// Layer
|
|
#[derive(Debug, PartialEq, Eq, Hash, Clone, PartialOrd, Ord)]
|
|
enum PositionLayer {
|
|
A,
|
|
B,
|
|
}
|
|
|
|
impl Display for PositionLayer {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
let c = match self {
|
|
PositionLayer::A => "A",
|
|
PositionLayer::B => "B",
|
|
};
|
|
write!(f, "{}", c)
|
|
}
|
|
}
|
|
|
|
impl FromStr for PositionLayer {
|
|
type Err = anyhow::Error;
|
|
|
|
fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
|
|
match s {
|
|
"A" => Ok(Self::A),
|
|
"B" => Ok(Self::B),
|
|
_ => Err(anyhow!("Invalid value")),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Where PS Board is placed while QAQC.
|
|
/// TODO: name
|
|
#[derive(Debug, PartialEq, Eq, Hash, Clone, PartialOrd, Ord)]
|
|
pub struct Position {
|
|
major: PositionLayer,
|
|
minor: u8,
|
|
patch: u8,
|
|
}
|
|
|
|
impl Display for Position {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
write!(f, "{}-{}-{}", self.major, self.minor, self.patch)
|
|
}
|
|
}
|
|
|
|
impl FromStr for Position {
|
|
type Err = anyhow::Error;
|
|
|
|
fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
|
|
let mut count = 0;
|
|
let mut major_pre = None;
|
|
let mut minor_pre = None;
|
|
let mut patch_pre = None;
|
|
for part in s.split('-') {
|
|
count += 1;
|
|
match count {
|
|
1 => major_pre = Some(part.parse()?),
|
|
2 => minor_pre = Some(part.parse()?),
|
|
3 => patch_pre = Some(part.parse()?),
|
|
_ => (),
|
|
}
|
|
}
|
|
if count > 3 {
|
|
return Err(anyhow!("Invalid Position format"));
|
|
}
|
|
|
|
Ok(Position {
|
|
major: major_pre.context("No major")?,
|
|
minor: minor_pre.context("No minor")?,
|
|
patch: patch_pre.context("No patch")?,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// PSB ID printed on the tape on the board.
|
|
#[derive(Debug, PartialEq, Clone)]
|
|
pub struct PsbId {
|
|
id: u32,
|
|
}
|
|
|
|
impl PsbId {
|
|
/// Without validation.
|
|
pub fn new(id: u32) -> Self {
|
|
PsbId { id }
|
|
}
|
|
}
|
|
|
|
impl FromStr for PsbId {
|
|
type Err = anyhow::Error;
|
|
|
|
fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
|
|
if !s.starts_with("PS") {
|
|
return Err(anyhow!("Must prefixed with PS: got: {}", s));
|
|
}
|
|
let num = s[2..].parse()?;
|
|
// TODO: add validation
|
|
warn!("No validation implemented");
|
|
Ok(PsbId::new(num))
|
|
}
|
|
}
|
|
|
|
/// All information on QAQC stored in the database.
|
|
///
|
|
/// Everything without `Option` is available from master log.
|
|
///
|
|
/// TODO: use pos? shifter? version?
|
|
#[serde_as]
|
|
#[derive(PartialEq, Debug, Serialize, Deserialize)]
|
|
pub struct PsbQaqcResult {
|
|
motherboard_id: u32,
|
|
daughterboard_id: Option<u32>,
|
|
#[serde_as(as = "DisplayFromStr")]
|
|
position: Position,
|
|
qspip: u8,
|
|
recov: u8,
|
|
power: u8,
|
|
clock: u8,
|
|
asdtp: u8,
|
|
reset: u16,
|
|
qaqc_result: u32,
|
|
lvds_tx_skew: Option<u32>,
|
|
runid: u32,
|
|
timestamp: DateTime<Utc>,
|
|
qaqc_log_file: String,
|
|
shiftscript_ver: Version,
|
|
shifter: String,
|
|
firmware_ver: Option<Version>,
|
|
parameter_ver: Option<Version>,
|
|
fpga_dna: Option<u64>,
|
|
retest: bool,
|
|
comment: String,
|
|
}
|
|
|
|
impl PsbQaqcResult {
|
|
/// Expand [`MasterLogResult`] to [`PsbQaqcResult`].
|
|
/// Filling unavailable fields with [`None`]s.
|
|
pub fn from_masterlogresult(result: MasterLogResult) -> Vec<Self> {
|
|
let mut converted = vec![];
|
|
for (pos, boardresult) in result.board_results {
|
|
let isretest = boardresult.is_need_retest().is_some();
|
|
let new = PsbQaqcResult {
|
|
motherboard_id: boardresult.id.id,
|
|
daughterboard_id: None,
|
|
position: pos,
|
|
qspip: boardresult.qspip,
|
|
recov: boardresult.recov,
|
|
power: boardresult.power,
|
|
clock: boardresult.clock,
|
|
asdtp: boardresult.asdtp,
|
|
reset: boardresult.reset,
|
|
qaqc_result: boardresult.result.into(),
|
|
lvds_tx_skew: None,
|
|
runid: result.runid,
|
|
timestamp: result.datetime,
|
|
qaqc_log_file: result.filename.clone(),
|
|
shiftscript_ver: result.version.clone(),
|
|
shifter: result.shifter.clone(),
|
|
firmware_ver: None,
|
|
parameter_ver: None,
|
|
fpga_dna: None,
|
|
retest: isretest,
|
|
comment: "".to_string(),
|
|
};
|
|
converted.push(new);
|
|
}
|
|
converted
|
|
}
|
|
}
|
|
|
|
// /// Iterator over output csv file but without results from abnormal resistance
|
|
// #[derive(Debug)]
|
|
// struct CsvMidReader<B> {
|
|
// lines: std::io::Lines<B>,
|
|
// /// skipped line numbers
|
|
// pub skipped: Vec<usize>,
|
|
// /// Current line count
|
|
// count: usize,
|
|
// linebuf: Vec<u8>,
|
|
// }
|
|
//
|
|
// impl<B> CsvMidReader<B> {
|
|
// pub fn new(lines: std::io::Lines<B>) -> Self {
|
|
// CsvMidReader {
|
|
// lines,
|
|
// skipped: vec![],
|
|
// count: 0,
|
|
// linebuf: b"".to_vec(),
|
|
// }
|
|
// }
|
|
// }
|
|
//
|
|
// impl<B: BufRead> Read for CsvMidReader<B> {
|
|
// fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
|
// let mut rest_buf_size = buf.len();
|
|
// let mut written_size = 0;
|
|
// while self.linebuf.len() <= rest_buf_size {
|
|
// trace!("linebuf: {:?}", self.linebuf.len());
|
|
// (0..(self.linebuf.len())).for_each(|i| {
|
|
// buf[i + written_size] = self.linebuf[i];
|
|
// });
|
|
// written_size += self.linebuf.len();
|
|
// rest_buf_size -= self.linebuf.len();
|
|
// self.linebuf = match self.lines.next() {
|
|
// None => {
|
|
// trace!("None");
|
|
// break;
|
|
// }
|
|
// Some(line) => {
|
|
// trace!("new line: {:?}", line);
|
|
// line?.as_bytes().to_vec()
|
|
// }
|
|
// };
|
|
// }
|
|
// if self.lines.next().is_some() {
|
|
// (0..rest_buf_size).for_each(|i| {
|
|
// buf[i + written_size] = self.linebuf[i];
|
|
// });
|
|
// self.linebuf.drain(0..rest_buf_size);
|
|
// written_size += rest_buf_size;
|
|
// } else {
|
|
// self.linebuf = b"".to_vec();
|
|
// }
|
|
// trace!("size: {}", written_size);
|
|
// Ok(written_size)
|
|
// }
|
|
// }
|
|
//
|
|
// impl<B: BufRead> Iterator for CsvMidReader<B> {
|
|
// type Item = std::result::Result<String, std::io::Error>;
|
|
//
|
|
// fn next(&mut self) -> Option<Self::Item> {
|
|
// loop {
|
|
// match self.lines.next() {
|
|
// Some(line) => {
|
|
// self.count += 1;
|
|
// match line {
|
|
// Ok(line) => match line.contains("abnormal resistance") {
|
|
// true => {
|
|
// self.skipped.push(self.count);
|
|
// continue;
|
|
// }
|
|
// false => return Some(Ok(line)),
|
|
// },
|
|
// Err(e) => return Some(Err(e.into())),
|
|
// }
|
|
// }
|
|
// None => return None,
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
//
|
|
// // impl<B> std::io::Read for CsvMidReader<B> {
|
|
// // fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
|
// // todo!()
|
|
// // }
|
|
// // }
|
|
// //
|
|
// fn filter_csv_lines(rdr: impl BufRead) -> Vec<String> {
|
|
// rdr.lines()
|
|
// .enumerate()
|
|
// .filter(|(num, line)| match line {
|
|
// Ok(line) => {
|
|
// if line.contains("abnormal resistance") {
|
|
// info!("Skipping line {}", num + 1);
|
|
// debug!("line: {}", line);
|
|
// false
|
|
// } else {
|
|
// true
|
|
// }
|
|
// }
|
|
// Err(e) => {
|
|
// error!("Error while reading: {}", e);
|
|
// false
|
|
// }
|
|
// })
|
|
// .map(|(_, line)| line.unwrap())
|
|
// .collect_vec()
|
|
// }
|
|
|
|
fn write_psbqaqc_csv(result: MasterLogResult, outfile: PathBuf) -> Result<()> {
|
|
let expanded_results = PsbQaqcResult::from_masterlogresult(result);
|
|
|
|
let mut wtr = match outfile.exists() {
|
|
true => {
|
|
let file = File::options()
|
|
.read(true)
|
|
.append(true)
|
|
.open(outfile.clone())?;
|
|
csv::WriterBuilder::new()
|
|
.has_headers(false)
|
|
.from_writer(file)
|
|
}
|
|
false => {
|
|
println!("Creating new file: {}", outfile.display());
|
|
let file = File::options()
|
|
.create_new(true)
|
|
.write(true)
|
|
.open(outfile.clone())?;
|
|
csv::WriterBuilder::new()
|
|
.has_headers(true)
|
|
.from_writer(file)
|
|
}
|
|
};
|
|
for result in expanded_results {
|
|
wtr.serialize(result)?;
|
|
}
|
|
wtr.flush()?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
let args = Args::parse();
|
|
env_logger::Builder::new()
|
|
.filter_level(args.verbose.log_level_filter())
|
|
.init();
|
|
debug!("Args: {:?}", args);
|
|
|
|
debug!(
|
|
"{:?}",
|
|
" 1".split_whitespace().next().unwrap().parse::<u8>()
|
|
);
|
|
|
|
match args.command {
|
|
Commands::AddMasterLog {
|
|
master_log,
|
|
outfile,
|
|
editor,
|
|
} => {
|
|
let result = MasterLogResult::parse_file(master_log)?;
|
|
debug!("{:?}", result);
|
|
|
|
// Print boards to retest
|
|
result.print_boards_to_retest(io::stdout())?;
|
|
|
|
let outfile = outfile.unwrap_or(get_output_filename(&result));
|
|
|
|
write_psbqaqc_csv(result, outfile.clone())?;
|
|
|
|
println!("Add comments about errors in the last column of the CSV file");
|
|
println!("Press any key to start editting...");
|
|
terminal::enable_raw_mode()?;
|
|
let _ = event::read()?;
|
|
terminal::disable_raw_mode()?;
|
|
|
|
let editor = match (editor, env::var("VISUAL"), env::var("EDITOR")) {
|
|
(Some(editor), _, _) => editor,
|
|
(None, Ok(editor), _) => {
|
|
info!("Use VISUAL");
|
|
editor
|
|
}
|
|
(None, Err(_), Ok(editor)) => {
|
|
info!("Use EDITOR");
|
|
editor
|
|
}
|
|
(None, Err(e1), Err(e2)) => {
|
|
info!("No VISUAL nor EDITOR found, {}, {}", e1, e2);
|
|
"nano".to_string()
|
|
}
|
|
};
|
|
std::process::Command::new(editor)
|
|
.arg(outfile.clone())
|
|
.spawn()?
|
|
.wait()?;
|
|
|
|
{
|
|
let f = File::open(outfile.clone())?;
|
|
let rdr = BufReader::new(f);
|
|
rdr.lines().for_each(|l| println!("{}", l.unwrap()));
|
|
}
|
|
println!();
|
|
println!("Copy the CSV above and paste it to the database(Google sheets)");
|
|
println!("Choose Data->Split text to columns to format it");
|
|
|
|
println!("Press any key when upload finished...");
|
|
terminal::enable_raw_mode()?;
|
|
let _ = event::read()?;
|
|
terminal::disable_raw_mode()?;
|
|
|
|
let uploaded_outfile = {
|
|
let mut new_file_name = outfile
|
|
.clone()
|
|
.file_stem()
|
|
.context("No file stem for out file")?
|
|
.to_owned();
|
|
new_file_name.push("_uploaded");
|
|
if let Some(ext) = outfile.extension() {
|
|
new_file_name.push(".");
|
|
new_file_name.push(ext);
|
|
};
|
|
PathBuf::from("log").join(new_file_name)
|
|
};
|
|
info!(
|
|
"Renaming {} to {}",
|
|
outfile.display(),
|
|
uploaded_outfile.is_dir()
|
|
);
|
|
fs::rename(outfile, uploaded_outfile)?;
|
|
}
|
|
Commands::ConvertMasterLog {
|
|
master_log,
|
|
outfile,
|
|
} => {
|
|
let result = MasterLogResult::parse_file(master_log)?;
|
|
debug!("{:?}", result);
|
|
|
|
// Print boards to retest
|
|
result.print_boards_to_retest(io::stdout())?;
|
|
|
|
let outfile = outfile.unwrap_or(get_output_filename(&result));
|
|
|
|
write_psbqaqc_csv(result, outfile)?;
|
|
}
|
|
Commands::CheckDB { csvfile } => {
|
|
// TODO: more friendly message (like row number)
|
|
// TODO: more validation options (e.g. for each stage(Master log, Slave log(end of
|
|
// campaign)))
|
|
// TODO: Masterログが得られたときや、行として完成していかないときなど複数段階のチェックを用意する
|
|
let file = File::options().read(true).open(csvfile.clone())?;
|
|
|
|
// tried to implement lazy evaluated `Read`er
|
|
// let bufrdr = BufReader::new(file);
|
|
// let skipped_reader = CsvMidReader::new(bufrdr.lines());
|
|
// let rdr = csv::Reader::from_reader(skipped_reader);
|
|
|
|
// filter and collect source beforehand
|
|
// let rdr = BufReader::new(file);
|
|
// let nl = "\n";
|
|
// let content = filter_csv_lines(rdr).join(nl);
|
|
// let content = content.as_bytes();
|
|
// let rdr = csv::Reader::from_reader(content);
|
|
|
|
let mut file_for_postfilter = {
|
|
let file = File::options().read(true).open(csvfile)?;
|
|
BufReader::new(file).lines()
|
|
};
|
|
let mut prev_postifilter_line = 0;
|
|
let rdr = csv::Reader::from_reader(file);
|
|
let mut isinvalid = false;
|
|
for (num, line) in rdr.into_deserialize::<PsbQaqcResult>().enumerate() {
|
|
let line = match line {
|
|
Ok(line) => Ok(line),
|
|
Err(err) => {
|
|
// catch
|
|
// line num is +1 because csv iterator doesn't include header line
|
|
match file_for_postfilter.nth(num + 1 - prev_postifilter_line) {
|
|
None => {
|
|
warn!("Error while reading file(line: {})", num + 1);
|
|
debug!("lines: {:?}", file_for_postfilter);
|
|
Err(err)
|
|
}
|
|
Some(line) => {
|
|
prev_postifilter_line = num + 2;
|
|
if let Ok(line) = line {
|
|
info!("error line on {}: line: {}", num, line);
|
|
if line.contains("abnormal resistance") {
|
|
println!("\"abnormal resistance\" at line {}", num + 1);
|
|
continue;
|
|
}
|
|
}
|
|
Err(err)
|
|
}
|
|
}
|
|
}
|
|
};
|
|
if line.is_err() {
|
|
isinvalid = true;
|
|
}
|
|
// increment the following num for *header line* + *0-origin offset*
|
|
let line = line.inspect_err(|e| error!("Invalid line at {}: {}", num + 2, e));
|
|
debug!("{}: line {:?}", num, line);
|
|
}
|
|
if isinvalid {
|
|
return Err(anyhow!("Invalid CSV format."));
|
|
}
|
|
}
|
|
Commands::AddSkew { logfile, outfile } => {
|
|
let skew = skew::parse_count_file(logfile).context("Error during getting skew")?;
|
|
trace!("Skew: {}", skew);
|
|
todo!("Not implemented")
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
use std::str::FromStr;
|
|
|
|
use chrono::DateTime;
|
|
|
|
use crate::{Position, PositionLayer};
|
|
|
|
#[test]
|
|
fn positionlayer_ordering() {
|
|
assert!(PositionLayer::A < PositionLayer::B)
|
|
}
|
|
|
|
#[test]
|
|
fn position_ordering() {
|
|
assert!(
|
|
Position {
|
|
major: PositionLayer::A,
|
|
minor: 1,
|
|
patch: 5
|
|
} < Position {
|
|
major: PositionLayer::A,
|
|
minor: 1,
|
|
patch: 7
|
|
}
|
|
);
|
|
assert!(
|
|
Position {
|
|
major: PositionLayer::A,
|
|
minor: 1,
|
|
patch: 5
|
|
} < Position {
|
|
major: PositionLayer::A,
|
|
minor: 2,
|
|
patch: 3
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_position() {
|
|
assert_eq!(
|
|
Position::from_str("A-1-4").unwrap(),
|
|
Position {
|
|
major: PositionLayer::A,
|
|
minor: 1,
|
|
patch: 4
|
|
}
|
|
);
|
|
assert_eq!(
|
|
Position::from_str("A-0-9").unwrap(),
|
|
Position {
|
|
major: PositionLayer::A,
|
|
minor: 0,
|
|
patch: 9
|
|
}
|
|
);
|
|
assert_ne!(
|
|
Position::from_str("A-1-4").unwrap(),
|
|
Position {
|
|
major: PositionLayer::A,
|
|
minor: 0,
|
|
patch: 2
|
|
}
|
|
);
|
|
assert_eq!(
|
|
Position::from_str("B-1-4").unwrap(),
|
|
Position {
|
|
major: PositionLayer::B,
|
|
minor: 1,
|
|
patch: 4
|
|
}
|
|
);
|
|
assert_eq!(
|
|
Position::from_str("B-0-9").unwrap(),
|
|
Position {
|
|
major: PositionLayer::B,
|
|
minor: 0,
|
|
patch: 9
|
|
}
|
|
);
|
|
assert_ne!(
|
|
Position::from_str("B-1-4").unwrap(),
|
|
Position {
|
|
major: PositionLayer::B,
|
|
minor: 0,
|
|
patch: 2
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_datetime() {
|
|
assert!(DateTime::parse_from_str("2024-06-20T08:42:01+0000", "%+").is_ok());
|
|
assert!(DateTime::parse_from_str("Date: 2024-06-20T08:42:01+0000", "Date: %+").is_ok());
|
|
}
|
|
|
|
// #[test]
|
|
// fn parse_file() {
|
|
// let logfile = r"""""";
|
|
// }
|
|
}
|