research/psboard-qaqc-postprocess
1
0
Fork 0
mirror of https://gitlab.cern.ch/wotsubo/psboard-qaqc-postprocess.git synced 2024-11-22 07:10:18 +09:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: separate master reading part as a mod

Browse source
This commit is contained in:
Wataru Otsubo 2024-07-22 16:02:34 +09:00
parent 2258067f48
commit b927f5923e
3 changed files with 319 additions and 294 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

296
src/main.rs
View file

@ -12,11 +12,13 @@ use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Local, Utc};
use clap::{Parser, Subcommand};
use log::{debug, error, info, trace, warn};
use masterlog::MasterLogResult;
use regex::Regex;
use semver::Version;
use serde::{Deserialize, Serialize};
use serde_with::{serde_as, DisplayFromStr};
mod masterlog;
mod skew;
/// PS Board QAQC shift related commands.
@ -153,243 +155,6 @@ impl FromStr for PsbId {
}
}
/// QAQC results for each boards extracted from master log.
///
/// Temporary stores raw values.
/// TODO: specify each filed types.(maybe multi values)
#[derive(Debug)]
struct MasterBoardResult {
id: PsbId,
qspip: u8,
recov: u8,
power: u8,
clock: u8,
asdtp: u8,
reset: u16,
result: u8,
}
/// Full result for a single QAQC run from a log file on JATHub master.
#[derive(Debug)]
pub struct MasterLogResult {
version: Version,
datetime: DateTime<Utc>,
shifter: String,
board_results: BTreeMap<Position, MasterBoardResult>,
filename: String,
}
/// Get version of shift script.
fn extract_version(line: &str) -> Result<Version> {
Ok(line
.split_whitespace()
.nth(2)
.context("Invalid log format(version)")?
.parse()?)
}
/// Get shifters from shifter line
fn extract_shifter_line(line: &str) -> Result<String> {
let re = Regex::new(r"^Shifters: (.+)$").unwrap();
let caps = re.captures(line).unwrap();
trace!("Regex {:?}", caps);
Ok(caps.get(1).unwrap().as_str().to_owned())
}
/// Get position and PSB ID pair from a master log line.
fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
let re = Regex::new(r"Position / assigned-ID : (.+) / (.+)").unwrap();
let caps = re.captures(line).context("No capture")?;
let pos = Position::from_str(caps.get(1).unwrap().into())?;
let id = PsbId::from_str(caps.get(2).unwrap().into())?;
Ok((pos, id))
}
impl MasterLogResult {
/// Parse log file on master jathub.
fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
let mut lines = file.lines();
let version = {
let line = lines.next().context("Invalid log format(no versions)")??;
extract_version(&line)?
};
let _sep = lines.next().context("Invalid log format(separator)")??;
let datetime: DateTime<Utc> = {
let line = lines
.next()
.context("Invalid log format(no datetime line)")??;
DateTime::parse_from_str(&line, "Date: %+")
.context("Invalid datetime format (must be ISO 8601)")?
.to_utc()
};
let shifter = {
let line = lines
.next()
.context("Invalid log format(no shifter line)")??;
extract_shifter_line(&line)?
};
let _sep = lines.next().context("Invalid log format")?;
if !lines
.next()
.context("Invalid log format")??
.starts_with("PBS Assignment:")
{
return Err(anyhow!("Invalid log format"));
}
let mut assignments = BTreeMap::new();
// till 19 for `===========`
for i in 0..19 {
let line = lines.next().context("Unexpected EOF")??;
if line.starts_with("=========") {
debug!("End of assignments");
break;
}
let (pos, id) = extract_position_id(&line)?;
match assignments.insert(pos.clone(), id) {
None => (),
Some(old_id) => {
return Err(anyhow!(
"Value already exists on row {}: {:?} => {:?}",
i,
pos,
old_id
));
}
};
}
trace!("Read all PBS assignments");
info!("{:?}", assignments);
// TODO: stricter validation for header?
if !lines
.next()
.context("Invalid log format")??
.contains("QAQC status")
{
info!("{:?}", lines.next());
return Err(anyhow!("Invalid log format(result table header)"));
}
let _sep = lines
.next()
.context("Invalid log format(result table separator)")??;
if !lines
.next()
.context("Invalid log format")??
.contains("Station0")
{
return Err(anyhow!("Invalid log format(result Station0)"));
}
let mut board_results = BTreeMap::new();
for station_minor in [0, 1] {
info!("Result for {:?}", station_minor);
for _ in 1..10 {
let line = lines.next().context("Invalid log format(result body)")??;
if line.contains("Station1") || line.contains("======") {
break;
}
let parts: Vec<&str> = line.split('|').collect();
let first = parts.first().context("No col 1")?;
let raw_station_id = {
let re = Regex::new(r"JATHub_( ?\d*)$")?;
re.captures(first).map(|v| {
v.get(1)
.unwrap()
.as_str()
.split_whitespace()
.next()
.unwrap()
.parse::<u8>()
})
}
.context("Invalid station format")??;
let station_id = match station_minor {
0 => raw_station_id,
1 => raw_station_id - 10,
_ => panic!("Unexpected"),
};
trace!("Row {} {:?}", station_id, parts);
if parts.len() != 9 {
return Err(anyhow!(
"Invalid number of results(expected: 9, detected: {})",
parts.len()
));
}
// Origin is different (-1)
let pos = Position {
major: PositionLayer::A,
minor: station_minor,
patch: station_id - 1,
};
debug!("pos from table {}", pos);
let psbid = assignments
.get(&pos)
.context(format!("No board on pos {}", pos))?
.clone();
let result = MasterBoardResult {
id: psbid,
qspip: parts
.get(1)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid qspip")??,
recov: parts
.get(2)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid recov")??,
power: parts
.get(3)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid power")??,
clock: parts
.get(4)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid clock")??,
asdtp: parts
.get(5)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid asdtp")??,
reset: parts
.get(6)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid reset")??,
result: parts
.get(7)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid result")??,
};
match board_results.insert(pos, result) {
None => (),
Some(v) => {
panic!("Unexpected value already exists: {:?}", v)
}
}
}
}
debug!("{:#?}", board_results);
Ok(MasterLogResult {
version,
datetime,
shifter,
board_results,
filename,
})
}
}
/// All information on QAQC stored in the database.
///
/// Everything without `Option` is available from master log.
@ -538,9 +303,7 @@ mod test {
use chrono::{DateTime, Utc};
use semver::Version;
use crate::{
extract_position_id, extract_shifter_line, extract_version, Position, PositionLayer, PsbId,
};
use crate::{Position, PositionLayer, PsbId};
#[test]
fn positionlayer_ordering() {
@ -607,59 +370,6 @@ mod test {
assert!(DateTime::parse_from_str("Date: 2024-06-20T08:42:01+0000", "Date: %+").is_ok());
}
#[test]
fn parse_version() {
assert_eq!(
extract_version("Shift script: 1.0.0").unwrap(),
Version::new(1, 0, 0)
)
}
#[test]
fn parse_shifter_line() {
assert_eq!(extract_shifter_line("Shifters: foo").unwrap(), "foo");
assert_eq!(
extract_shifter_line("Shifters: foo bar").unwrap(),
"foo bar"
);
}
#[test]
fn parse_pos_id_line() {
assert_eq!(
extract_position_id("Position / assigned-ID : A-0-0 / PS0004").unwrap(),
(
Position {
major: PositionLayer::A,
minor: 0,
patch: 0,
},
PsbId::new(4)
)
);
assert_eq!(
extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(),
(
Position {
major: PositionLayer::A,
minor: 1,
patch: 7,
},
PsbId::new(108)
)
);
assert_ne!(
extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(),
(
Position {
major: PositionLayer::A,
minor: 0,
patch: 7,
},
PsbId::new(106)
)
);
}
// #[test]
// fn parse_file() {

309
src/masterlog.rs Normal file
View file

@ -0,0 +1,309 @@
use std::{collections::BTreeMap, io::BufRead, str::FromStr};
use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Utc};
use log::{debug, info, trace};
use regex::Regex;
use semver::Version;
use crate::{Position, PositionLayer, PsbId};
/// QAQC results for each boards extracted from master log.
///
/// Temporary stores raw values.
/// TODO: specify each filed types.(maybe multi values)
#[derive(Debug)]
pub struct MasterBoardResult {
pub id: PsbId,
pub qspip: u8,
pub recov: u8,
pub power: u8,
pub clock: u8,
pub asdtp: u8,
pub reset: u16,
pub result: u8,
}
/// Full result for a single QAQC run from a log file on JATHub master.
#[derive(Debug)]
pub struct MasterLogResult {
pub version: Version,
pub datetime: DateTime<Utc>,
pub shifter: String,
pub board_results: BTreeMap<Position, MasterBoardResult>,
pub filename: String,
}
/// Get version of shift script.
fn extract_version(line: &str) -> Result<Version> {
Ok(line
.split_whitespace()
.nth(2)
.context("Invalid log format(version)")?
.parse()?)
}
/// Get shifters from shifter line
fn extract_shifter_line(line: &str) -> Result<String> {
let re = Regex::new(r"^Shifters: (.+)$").unwrap();
let caps = re.captures(line).unwrap();
trace!("Regex {:?}", caps);
Ok(caps.get(1).unwrap().as_str().to_owned())
}
/// Get position and PSB ID pair from a master log line.
fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
let re = Regex::new(r"Position / assigned-ID : (.+) / (.+)").unwrap();
let caps = re.captures(line).context("No capture")?;
let pos = Position::from_str(caps.get(1).unwrap().into())?;
let id = PsbId::from_str(caps.get(2).unwrap().into())?;
Ok((pos, id))
}
impl MasterLogResult {
/// Parse log file on master jathub.
pub fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
let mut lines = file.lines();
let version = {
let line = lines.next().context("Invalid log format(no versions)")??;
extract_version(&line)?
};
let _sep = lines.next().context("Invalid log format(separator)")??;
let datetime: DateTime<Utc> = {
let line = lines
.next()
.context("Invalid log format(no datetime line)")??;
DateTime::parse_from_str(&line, "Date: %+")
.context("Invalid datetime format (must be ISO 8601)")?
.to_utc()
};
let shifter = {
let line = lines
.next()
.context("Invalid log format(no shifter line)")??;
extract_shifter_line(&line)?
};
let _sep = lines.next().context("Invalid log format")?;
if !lines
.next()
.context("Invalid log format")??
.starts_with("PBS Assignment:")
{
return Err(anyhow!("Invalid log format"));
}
let mut assignments = BTreeMap::new();
// till 19 for `===========`
for i in 0..19 {
let line = lines.next().context("Unexpected EOF")??;
if line.starts_with("=========") {
debug!("End of assignments");
break;
}
let (pos, id) = extract_position_id(&line)?;
match assignments.insert(pos.clone(), id) {
None => (),
Some(old_id) => {
return Err(anyhow!(
"Value already exists on row {}: {:?} => {:?}",
i,
pos,
old_id
));
}
};
}
trace!("Read all PBS assignments");
info!("{:?}", assignments);
// TODO: stricter validation for header?
if !lines
.next()
.context("Invalid log format")??
.contains("QAQC status")
{
info!("{:?}", lines.next());
return Err(anyhow!("Invalid log format(result table header)"));
}
let _sep = lines
.next()
.context("Invalid log format(result table separator)")??;
if !lines
.next()
.context("Invalid log format")??
.contains("Station0")
{
return Err(anyhow!("Invalid log format(result Station0)"));
}
let mut board_results = BTreeMap::new();
for station_minor in [0, 1] {
info!("Result for {:?}", station_minor);
for _ in 1..10 {
let line = lines.next().context("Invalid log format(result body)")??;
if line.contains("Station1") || line.contains("======") {
break;
}
let parts: Vec<&str> = line.split('|').collect();
let first = parts.first().context("No col 1")?;
let raw_station_id = {
let re = Regex::new(r"JATHub_( ?\d*)$")?;
re.captures(first).map(|v| {
v.get(1)
.unwrap()
.as_str()
.split_whitespace()
.next()
.unwrap()
.parse::<u8>()
})
}
.context("Invalid station format")??;
let station_id = match station_minor {
0 => raw_station_id,
1 => raw_station_id - 10,
_ => panic!("Unexpected"),
};
trace!("Row {} {:?}", station_id, parts);
if parts.len() != 9 {
return Err(anyhow!(
"Invalid number of results(expected: 9, detected: {})",
parts.len()
));
}
// Origin is different (-1)
let pos = Position {
major: PositionLayer::A,
minor: station_minor,
patch: station_id - 1,
};
debug!("pos from table {}", pos);
let psbid = assignments
.get(&pos)
.context(format!("No board on pos {}", pos))?
.clone();
let result = MasterBoardResult {
id: psbid,
qspip: parts
.get(1)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid qspip")??,
recov: parts
.get(2)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid recov")??,
power: parts
.get(3)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid power")??,
clock: parts
.get(4)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid clock")??,
asdtp: parts
.get(5)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid asdtp")??,
reset: parts
.get(6)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid reset")??,
result: parts
.get(7)
.map(|v| v.split_whitespace().next().unwrap().parse())
.context("Invalid result")??,
};
match board_results.insert(pos, result) {
None => (),
Some(v) => {
panic!("Unexpected value already exists: {:?}", v)
}
}
}
}
debug!("{:#?}", board_results);
Ok(MasterLogResult {
version,
datetime,
shifter,
board_results,
filename,
})
}
}
#[cfg(test)]
mod test {
use semver::Version;
use super::{Position, PositionLayer, PsbId};
use super::{extract_position_id, extract_shifter_line, extract_version};
#[test]
fn parse_version() {
assert_eq!(
extract_version("Shift script: 1.0.0").unwrap(),
Version::new(1, 0, 0)
)
}
#[test]
fn parse_shifter_line() {
assert_eq!(extract_shifter_line("Shifters: foo").unwrap(), "foo");
assert_eq!(
extract_shifter_line("Shifters: foo bar").unwrap(),
"foo bar"
);
}
#[test]
fn parse_pos_id_line() {
assert_eq!(
extract_position_id("Position / assigned-ID : A-0-0 / PS0004").unwrap(),
(
Position {
major: PositionLayer::A,
minor: 0,
patch: 0,
},
PsbId::new(4)
)
);
assert_eq!(
extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(),
(
Position {
major: PositionLayer::A,
minor: 1,
patch: 7,
},
PsbId::new(108)
)
);
assert_ne!(
extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(),
(
Position {
major: PositionLayer::A,
minor: 0,
patch: 7,
},
PsbId::new(106)
)
);
}
}

8
tests/cli.rs
View file

@ -30,7 +30,13 @@ mod integrated_test {
let r = BufReader::new(f);
assert!(r
.lines()
.any(|line| { line.unwrap().contains("8868,,A-0-1,0,1,1,0,1,8,1,") }));
.any(|line| {
line
.unwrap()
.contains(
"8866,,A-0-0,0,1,1,0,1,8,1,,2024-07-20T17:15:46Z,20240720_171418.log,0.1.0,alice,,,,"
)
}));
}
// 2nd file