use core::{panic, str}; use std::{ collections::HashMap, fmt::Display, fs::File, io::{BufRead, BufReader}, path::{self, PathBuf}, str::FromStr, }; use anyhow::{anyhow, Context, Result}; use chrono::{DateTime, Local, Utc}; use clap::Parser; use log::{debug, error, info, trace, warn}; use regex::Regex; use semver::Version; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; /// Parse master jathub logfile for PS Board QAQC and write out to CSV. #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Args { /// Master log file. master_log: path::PathBuf, /// Output CSV file. // #[arg(default_value = get_default_log_path().into_os_string())] outfile: PathBuf, #[command(flatten)] verbose: clap_verbosity_flag::Verbosity, } /// Layer #[derive(Debug, PartialEq, Eq, Hash, Clone)] enum PositionLayer { A, B, } impl Display for PositionLayer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let c = match self { PositionLayer::A => "A", PositionLayer::B => "B", }; write!(f, "{}", c) } } impl FromStr for PositionLayer { type Err = anyhow::Error; fn from_str(s: &str) -> std::result::Result { match s { "A" => Ok(Self::A), "B" => Ok(Self::B), _ => Err(anyhow!("Invalid value")), } } } /// Where PS Board is placed while QAQC. /// TODO: name #[derive(Debug, PartialEq, Eq, Hash, Clone)] struct Position { major: PositionLayer, minor: u8, patch: u8, } impl Display for Position { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}-{}-{}", self.major, self.minor, self.patch) } } impl FromStr for Position { type Err = anyhow::Error; fn from_str(s: &str) -> std::result::Result { let mut count = 0; let mut major_pre = None; let mut minor_pre = None; let mut patch_pre = None; for part in s.split('-') { count += 1; match count { 1 => major_pre = Some(part.parse()?), 2 => minor_pre = Some(part.parse()?), 3 => patch_pre = Some(part.parse()?), _ => (), } } if count > 3 { return Err(anyhow!("Invalid Position format")); } Ok(Position { major: major_pre.context("No major")?, minor: minor_pre.context("No minor")?, patch: patch_pre.context("No patch")?, }) } } /// PSB ID printed on the tape on the board. #[derive(Debug, PartialEq, Clone)] struct PsbId { id: u32, } impl PsbId { /// Without validation. pub fn new(id: u32) -> Self { PsbId { id } } } impl FromStr for PsbId { type Err = anyhow::Error; fn from_str(s: &str) -> std::result::Result { if !s.starts_with("PS") { return Err(anyhow!("Must prefixed with PS: got: {}", s)); } let num = s[2..].parse()?; // TODO: add validation warn!("No validation implemented"); Ok(PsbId::new(num)) } } /// QAQC results for each boards extracted from master log. /// /// Temporary stores raw values. /// TODO: specify each filed types.(maybe multi values) #[derive(Debug)] struct MasterBoardResult { id: PsbId, qspip: u8, recov: u8, power: u8, clock: u8, asdtp: u8, reset: u16, result: u8, } /// Full result for a single QAQC run from a log file on JATHub master. #[derive(Debug)] pub struct MasterLogResult { version: Version, datetime: DateTime, shifter: String, board_results: HashMap, filename: String, } /// Get version of shift script. fn extract_version(line: &str) -> Result { Ok(line .split_whitespace() .nth(2) .context("Invalid log format(version)")? .parse()?) } /// Get shifters from shifter line fn extract_shifter_line(line: &str) -> Result { let re = Regex::new(r"^Shifters: (.+)$").unwrap(); let caps = re.captures(line).unwrap(); trace!("Regex {:?}", caps); Ok(caps.get(1).unwrap().as_str().to_owned()) } /// Get position and PSB ID pair from a master log line. fn extract_position_id(line: &str) -> Result<(Position, PsbId)> { let re = Regex::new(r"Position / assigned-ID : (.+) / (.+)").unwrap(); let caps = re.captures(line).context("No capture")?; let pos = Position::from_str(caps.get(1).unwrap().into())?; let id = PsbId::from_str(caps.get(2).unwrap().into())?; Ok((pos, id)) } impl MasterLogResult { /// Parse log file on master jathub. fn parse_file(file: impl BufRead, filename: String) -> Result { let mut lines = file.lines(); let version = { let line = lines.next().context("Invalid log format(no versions)")??; extract_version(&line)? }; let _sep = lines.next().context("Invalid log format(separator)")??; let datetime: DateTime = { let line = lines .next() .context("Invalid log format(no datetime line)")??; DateTime::parse_from_str(&line, "Date: %+") .context("Invalid datetime format (must be ISO 8601)")? .to_utc() }; let shifter = { let line = lines .next() .context("Invalid log format(no shifter line)")??; extract_shifter_line(&line)? }; let _sep = lines.next().context("Invalid log format")?; if !lines .next() .context("Invalid log format")?? .starts_with("PBS Assignment:") { return Err(anyhow!("Invalid log format")); } let mut assignments = HashMap::new(); // till 19 for `===========` for i in 0..19 { let line = lines.next().context("Unexpected EOF")??; if line.starts_with("=========") { debug!("End of assignments"); break; } let (pos, id) = extract_position_id(&line)?; match assignments.insert(pos.clone(), id) { None => (), Some(old_id) => { return Err(anyhow!( "Value already exists on row {}: {:?} => {:?}", i, pos, old_id )); } }; } trace!("Read all PBS assignments"); info!("{:?}", assignments); // TODO: stricter validation for header? if !lines .next() .context("Invalid log format")?? .contains("QAQC status") { info!("{:?}", lines.next()); return Err(anyhow!("Invalid log format(result table header)")); } let _sep = lines .next() .context("Invalid log format(result table separator)")??; if !lines .next() .context("Invalid log format")?? .contains("Station0") { return Err(anyhow!("Invalid log format(result Station0)")); } let mut board_results = HashMap::new(); for station_minor in [0, 1] { info!("Result for {:?}", station_minor); for _ in 1..10 { let line = lines.next().context("Invalid log format(result body)")??; if line.contains("Station1") || line.contains("======") { break; } let parts: Vec<&str> = line.split('|').collect(); let first = parts.first().context("No col 1")?; let raw_station_id = { let re = Regex::new(r"JATHub_( ?\d*)$")?; re.captures(first).map(|v| { v.get(1) .unwrap() .as_str() .split_whitespace() .next() .unwrap() .parse::() }) } .context("Invalid station format")??; let station_id = match station_minor { 0 => raw_station_id, 1 => raw_station_id - 10, _ => panic!("Unexpected"), }; trace!("Row {} {:?}", station_id, parts); if parts.len() != 9 { return Err(anyhow!( "Invalid number of results(expected: 9, detected: {})", parts.len() )); } // Origin is different (-1) let pos = Position { major: PositionLayer::A, minor: station_minor, patch: station_id - 1, }; debug!("pos from table {}", pos); let psbid = assignments .get(&pos) .context(format!("No board on pos {}", pos))? .clone(); let result = MasterBoardResult { id: psbid, qspip: parts .get(1) .map(|v| v.split_whitespace().next().unwrap().parse()) .context("Invalid qspip")??, recov: parts .get(2) .map(|v| v.split_whitespace().next().unwrap().parse()) .context("Invalid recov")??, power: parts .get(3) .map(|v| v.split_whitespace().next().unwrap().parse()) .context("Invalid power")??, clock: parts .get(4) .map(|v| v.split_whitespace().next().unwrap().parse()) .context("Invalid clock")??, asdtp: parts .get(5) .map(|v| v.split_whitespace().next().unwrap().parse()) .context("Invalid asdtp")??, reset: parts .get(6) .map(|v| v.split_whitespace().next().unwrap().parse()) .context("Invalid reset")??, result: parts .get(7) .map(|v| v.split_whitespace().next().unwrap().parse()) .context("Invalid result")??, }; match board_results.insert(pos, result) { None => (), Some(v) => { panic!("Unexpected value already exists: {:?}", v) } } } } debug!("{:#?}", board_results); Ok(MasterLogResult { version, datetime, shifter, board_results, filename, }) } } /// All information on QAQC stored in the database. /// /// Everything without `Option` is available from master log. /// /// TODO: use pos? shifter? version? #[serde_as] #[derive(PartialEq, Debug, Serialize, Deserialize)] pub struct PsbQaqcResult { motherboard_id: u32, daughterboard_id: Option, #[serde_as(as = "DisplayFromStr")] position: Position, qspip: u8, recov: u8, power: u8, clock: u8, asdtp: u8, reset: u16, qaqc_result: u32, lvds_tx_skew: Option, ppl_lock_reset_count: Option, timestamp: DateTime, qaqc_log_file: String, shiftscript_ver: Version, shifter: String, firmware_ver: Option, parameter_ver: Option, fpga_dna: Option, comment: String, } impl PsbQaqcResult { /// Expand [`MasterLogResult`] to [`PsbQaqcResult`]. /// Filling unavailable fields with [`None`]s. pub fn from_masterlogresult(result: MasterLogResult) -> Vec { let mut converted = vec![]; for (pos, boardresult) in result.board_results { let new = PsbQaqcResult { motherboard_id: boardresult.id.id, daughterboard_id: None, position: pos, qspip: boardresult.qspip, recov: boardresult.recov, power: boardresult.power, clock: boardresult.clock, asdtp: boardresult.asdtp, reset: boardresult.reset, qaqc_result: boardresult.result.into(), lvds_tx_skew: None, ppl_lock_reset_count: None, timestamp: result.datetime, qaqc_log_file: result.filename.clone(), shiftscript_ver: result.version.clone(), shifter: result.shifter.clone(), firmware_ver: None, parameter_ver: None, fpga_dna: None, comment: "".to_string(), }; converted.push(new); } converted } } fn main() -> Result<()> { let args = Args::parse(); env_logger::Builder::new() .filter_level(args.verbose.log_level_filter()) .init(); debug!("Args: {:?}", args); debug!( "{:?}", " 1".split_whitespace().next().unwrap().parse::() ); let result = { let file = File::open(args.master_log.clone())?; let reader = BufReader::new(file); MasterLogResult::parse_file( reader, args.master_log .file_name() .unwrap() .to_str() .unwrap() .to_string(), )? }; debug!("{:?}", result); // { // let file = File::options() // .read(true) // .open(args.outfile)?; // // let mut rdr = csv::Reader::from_reader(file); // rdr.records(); // } let expanded_results = PsbQaqcResult::from_masterlogresult(result); let mut wtr = match args.outfile.exists() { true => { let file = File::options().read(true).append(true).open(args.outfile)?; csv::WriterBuilder::new() .has_headers(false) .from_writer(file) } false => { println!("Creating new file: {}", args.outfile.display()); let file = File::options() .create_new(true) .write(true) .open(args.outfile)?; csv::WriterBuilder::new() .has_headers(true) .from_writer(file) } }; for result in expanded_results { wtr.serialize(result)?; } wtr.flush()?; Ok(()) } #[cfg(test)] mod test { use std::str::FromStr; use chrono::{DateTime, Utc}; use semver::Version; use crate::{ extract_position_id, extract_shifter_line, extract_version, Position, PositionLayer, PsbId, }; #[test] fn parse_position() { assert_eq!( Position::from_str("A-1-4").unwrap(), Position { major: PositionLayer::A, minor: 1, patch: 4 } ); assert_eq!( Position::from_str("A-0-9").unwrap(), Position { major: PositionLayer::A, minor: 0, patch: 9 } ); assert_ne!( Position::from_str("A-1-4").unwrap(), Position { major: PositionLayer::A, minor: 0, patch: 2 } ); } #[test] fn parse_datetime() { assert!(DateTime::parse_from_str("2024-06-20T08:42:01+0000", "%+").is_ok()); assert!(DateTime::parse_from_str("Date: 2024-06-20T08:42:01+0000", "Date: %+").is_ok()); } #[test] fn parse_version() { assert_eq!( extract_version("Shift script: 1.0.0").unwrap(), Version::new(1, 0, 0) ) } #[test] fn parse_shifter_line() { assert_eq!(extract_shifter_line("Shifters: foo").unwrap(), "foo"); assert_eq!( extract_shifter_line("Shifters: foo bar").unwrap(), "foo bar" ); } #[test] fn parse_pos_id_line() { assert_eq!( extract_position_id("Position / assigned-ID : A-0-0 / PS0004").unwrap(), ( Position { major: PositionLayer::A, minor: 0, patch: 0, }, PsbId::new(4) ) ); assert_eq!( extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(), ( Position { major: PositionLayer::A, minor: 1, patch: 7, }, PsbId::new(108) ) ); assert_ne!( extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(), ( Position { major: PositionLayer::A, minor: 0, patch: 7, }, PsbId::new(106) ) ); } }