psboard-qaqc-postprocess/src/main.rs

use core::{panic, str};
use std::{
    collections::HashMap,
    fmt::Display,
    fs::File,
    io::{BufRead, BufReader},
    path::{self, PathBuf},
    str::FromStr,
};

use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Local, Utc};
use clap::Parser;
use log::{debug, error, info, trace, warn};
use regex::Regex;
use semver::Version;
use serde::{Deserialize, Serialize};
use serde_with::{serde_as, DisplayFromStr};

/// Parse master jathub logfile for PS Board QAQC and write out to CSV.
#[derive(Parser, Debug)]
#[command(version, about, long_about = None)]
struct Args {
    /// Master log file.
    master_log: path::PathBuf,
    /// Output CSV file.
    // #[arg(default_value = get_default_log_path().into_os_string())]
    outfile: PathBuf,
    #[command(flatten)]
    verbose: clap_verbosity_flag::Verbosity,
}

/// Layer
#[derive(Debug, PartialEq, Eq, Hash, Clone)]
enum PositionLayer {
    A,
    B,
}

impl Display for PositionLayer {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let c = match self {
            PositionLayer::A => "A",
            PositionLayer::B => "B",
        };
        write!(f, "{}", c)
    }
}

impl FromStr for PositionLayer {
    type Err = anyhow::Error;

    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        match s {
            "A" => Ok(Self::A),
            "B" => Ok(Self::B),
            _ => Err(anyhow!("Invalid value")),
        }
    }
}

/// Where PS Board is placed while QAQC.
/// TODO: name
#[derive(Debug, PartialEq, Eq, Hash, Clone)]
struct Position {
    major: PositionLayer,
    minor: u8,
    patch: u8,
}

impl Display for Position {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}-{}-{}", self.major, self.minor, self.patch)
    }
}

impl FromStr for Position {
    type Err = anyhow::Error;

    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        let mut count = 0;
        let mut major_pre = None;
        let mut minor_pre = None;
        let mut patch_pre = None;
        for part in s.split('-') {
            count += 1;
            match count {
                1 => major_pre = Some(part.parse()?),
                2 => minor_pre = Some(part.parse()?),
                3 => patch_pre = Some(part.parse()?),
                _ => (),
            }
        }
        if count > 3 {
            return Err(anyhow!("Invalid Position format"));
        }

        Ok(Position {
            major: major_pre.context("No major")?,
            minor: minor_pre.context("No minor")?,
            patch: patch_pre.context("No patch")?,
        })
    }
}

/// PSB ID printed on the tape on the board.
#[derive(Debug, PartialEq, Clone)]
struct PsbId {
    id: u32,
}

impl PsbId {
    /// Without validation.
    pub fn new(id: u32) -> Self {
        PsbId { id }
    }
}

impl FromStr for PsbId {
    type Err = anyhow::Error;

    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        if !s.starts_with("PS") {
            return Err(anyhow!("Must prefixed with PS: got: {}", s));
        }
        let num = s[2..].parse()?;
        // TODO: add validation
        warn!("No validation implemented");
        Ok(PsbId::new(num))
    }
}

/// QAQC results for each boards extracted from master log.
///
/// Temporary stores raw values.
/// TODO: specify each filed types.(maybe multi values)
#[derive(Debug)]
struct MasterBoardResult {
    id: PsbId,
    qspip: u8,
    recov: u8,
    power: u8,
    clock: u8,
    asdtp: u8,
    reset: u16,
    result: u8,
}

/// Full result for a single QAQC run from a log file on JATHub master.
#[derive(Debug)]
pub struct MasterLogResult {
    version: Version,
    datetime: DateTime<Utc>,
    shifter: String,
    board_results: HashMap<Position, MasterBoardResult>,
    filename: String,
}

/// Get version of shift script.
fn extract_version(line: &str) -> Result<Version> {
    Ok(line
        .split_whitespace()
        .nth(2)
        .context("Invalid log format(version)")?
        .parse()?)
}

/// Get shifters from shifter line
fn extract_shifter_line(line: &str) -> Result<String> {
    let re = Regex::new(r"^Shifters: (.+)$").unwrap();
    let caps = re.captures(line).unwrap();
    trace!("Regex {:?}", caps);
    Ok(caps.get(1).unwrap().as_str().to_owned())
}

/// Get position and PSB ID pair from a master log line.
fn extract_position_id(line: &str) -> Result<(Position, PsbId)> {
    let re = Regex::new(r"Position / assigned-ID : (.+) / (.+)").unwrap();
    let caps = re.captures(line).context("No capture")?;
    let pos = Position::from_str(caps.get(1).unwrap().into())?;
    let id = PsbId::from_str(caps.get(2).unwrap().into())?;
    Ok((pos, id))
}

impl MasterLogResult {
    /// Parse log file on master jathub.
    fn parse_file(file: impl BufRead, filename: String) -> Result<MasterLogResult> {
        let mut lines = file.lines();

        let version = {
            let line = lines.next().context("Invalid log format(no versions)")??;
            extract_version(&line)?
        };

        let _sep = lines.next().context("Invalid log format(separator)")??;

        let datetime: DateTime<Utc> = {
            let line = lines
                .next()
                .context("Invalid log format(no datetime line)")??;
            DateTime::parse_from_str(&line, "Date: %+")
                .context("Invalid datetime format (must be ISO 8601)")?
                .to_utc()
        };

        let shifter = {
            let line = lines
                .next()
                .context("Invalid log format(no shifter line)")??;
            extract_shifter_line(&line)?
        };

        let _sep = lines.next().context("Invalid log format")?;

        if !lines
            .next()
            .context("Invalid log format")??
            .starts_with("PBS Assignment:")
        {
            return Err(anyhow!("Invalid log format"));
        }

        let mut assignments = HashMap::new();
        // till 19 for `===========`
        for i in 0..19 {
            let line = lines.next().context("Unexpected EOF")??;
            if line.starts_with("=========") {
                debug!("End of assignments");
                break;
            }
            let (pos, id) = extract_position_id(&line)?;
            match assignments.insert(pos.clone(), id) {
                None => (),
                Some(old_id) => {
                    return Err(anyhow!(
                        "Value already exists on row {}: {:?} => {:?}",
                        i,
                        pos,
                        old_id
                    ));
                }
            };
        }
        trace!("Read all PBS assignments");
        info!("{:?}", assignments);

        // TODO: stricter validation for header?
        if !lines
            .next()
            .context("Invalid log format")??
            .contains("QAQC status")
        {
            info!("{:?}", lines.next());
            return Err(anyhow!("Invalid log format(result table header)"));
        }
        let _sep = lines
            .next()
            .context("Invalid log format(result table separator)")??;

        if !lines
            .next()
            .context("Invalid log format")??
            .contains("Station0")
        {
            return Err(anyhow!("Invalid log format(result Station0)"));
        }

        let mut board_results = HashMap::new();
        for station_minor in [0, 1] {
            info!("Result for {:?}", station_minor);
            for _ in 1..10 {
                let line = lines.next().context("Invalid log format(result body)")??;
                if line.contains("Station1") || line.contains("======") {
                    break;
                }
                let parts: Vec<&str> = line.split('|').collect();
                let first = parts.first().context("No col 1")?;
                let raw_station_id = {
                    let re = Regex::new(r"JATHub_( ?\d*)$")?;
                    re.captures(first).map(|v| {
                        v.get(1)
                            .unwrap()
                            .as_str()
                            .split_whitespace()
                            .next()
                            .unwrap()
                            .parse::<u8>()
                    })
                }
                .context("Invalid station format")??;
                let station_id = match station_minor {
                    0 => raw_station_id,
                    1 => raw_station_id - 10,
                    _ => panic!("Unexpected"),
                };
                trace!("Row {} {:?}", station_id, parts);
                if parts.len() != 9 {
                    return Err(anyhow!(
                        "Invalid number of results(expected: 9, detected: {})",
                        parts.len()
                    ));
                }

                // Origin is different (-1)
                let pos = Position {
                    major: PositionLayer::A,
                    minor: station_minor,
                    patch: station_id - 1,
                };
                debug!("pos from table {}", pos);

                let psbid = assignments
                    .get(&pos)
                    .context(format!("No board on pos {}", pos))?
                    .clone();

                let result = MasterBoardResult {
                    id: psbid,
                    qspip: parts
                        .get(1)
                        .map(|v| v.split_whitespace().next().unwrap().parse())
                        .context("Invalid qspip")??,
                    recov: parts
                        .get(2)
                        .map(|v| v.split_whitespace().next().unwrap().parse())
                        .context("Invalid recov")??,
                    power: parts
                        .get(3)
                        .map(|v| v.split_whitespace().next().unwrap().parse())
                        .context("Invalid power")??,
                    clock: parts
                        .get(4)
                        .map(|v| v.split_whitespace().next().unwrap().parse())
                        .context("Invalid clock")??,
                    asdtp: parts
                        .get(5)
                        .map(|v| v.split_whitespace().next().unwrap().parse())
                        .context("Invalid asdtp")??,
                    reset: parts
                        .get(6)
                        .map(|v| v.split_whitespace().next().unwrap().parse())
                        .context("Invalid reset")??,
                    result: parts
                        .get(7)
                        .map(|v| v.split_whitespace().next().unwrap().parse())
                        .context("Invalid result")??,
                };

                match board_results.insert(pos, result) {
                    None => (),
                    Some(v) => {
                        panic!("Unexpected value already exists: {:?}", v)
                    }
                }
            }
        }

        debug!("{:#?}", board_results);

        Ok(MasterLogResult {
            version,
            datetime,
            shifter,
            board_results,
            filename,
        })
    }
}

/// All information on QAQC stored in the database.
///
/// Everything without `Option` is available from master log.
///
/// TODO: use pos? shifter? version?
#[serde_as]
#[derive(PartialEq, Debug, Serialize, Deserialize)]
pub struct PsbQaqcResult {
    motherboard_id: u32,
    daughterboard_id: Option<u32>,
    #[serde_as(as = "DisplayFromStr")]
    position: Position,
    qspip: u8,
    recov: u8,
    power: u8,
    clock: u8,
    asdtp: u8,
    reset: u16,
    qaqc_result: u32,
    lvds_tx_skew: Option<u32>,
    ppl_lock_reset_count: Option<i32>,
    timestamp: DateTime<Utc>,
    qaqc_log_file: String,
    shiftscript_ver: Version,
    shifter: String,
    firmware_ver: Option<Version>,
    parameter_ver: Option<Version>,
    fpga_dna: Option<u64>,
    comment: String,
}

impl PsbQaqcResult {
    /// Expand [`MasterLogResult`] to [`PsbQaqcResult`].
    /// Filling unavailable fields with [`None`]s.
    pub fn from_masterlogresult(result: MasterLogResult) -> Vec<Self> {
        let mut converted = vec![];
        for (pos, boardresult) in result.board_results {
            let new = PsbQaqcResult {
                motherboard_id: boardresult.id.id,
                daughterboard_id: None,
                position: pos,
                qspip: boardresult.qspip,
                recov: boardresult.recov,
                power: boardresult.power,
                clock: boardresult.clock,
                asdtp: boardresult.asdtp,
                reset: boardresult.reset,
                qaqc_result: boardresult.result.into(),
                lvds_tx_skew: None,
                ppl_lock_reset_count: None,
                timestamp: result.datetime,
                qaqc_log_file: result.filename.clone(),
                shiftscript_ver: result.version.clone(),
                shifter: result.shifter.clone(),
                firmware_ver: None,
                parameter_ver: None,
                fpga_dna: None,
                comment: "".to_string(),
            };
            converted.push(new);
        }
        converted
    }
}

fn main() -> Result<()> {
    let args = Args::parse();
    env_logger::Builder::new()
        .filter_level(args.verbose.log_level_filter())
        .init();
    debug!("Args: {:?}", args);

    debug!(
        "{:?}",
        "    1".split_whitespace().next().unwrap().parse::<u8>()
    );

    let result = {
        let file = File::open(args.master_log.clone())?;
        let reader = BufReader::new(file);
        MasterLogResult::parse_file(
            reader,
            args.master_log
                .file_name()
                .unwrap()
                .to_str()
                .unwrap()
                .to_string(),
        )?
    };
    debug!("{:?}", result);

    // {
    //     let file = File::options()
    //         .read(true)
    //         .open(args.outfile)?;
    //
    //     let mut rdr = csv::Reader::from_reader(file);
    //     rdr.records();
    // }

    let expanded_results = PsbQaqcResult::from_masterlogresult(result);

    let mut wtr = match args.outfile.exists() {
        true => {
            let file = File::options().read(true).append(true).open(args.outfile)?;
            csv::WriterBuilder::new()
                .has_headers(false)
                .from_writer(file)
        }
        false => {
            println!("Creating new file: {}", args.outfile.display());
            let file = File::options()
                .create_new(true)
                .write(true)
                .open(args.outfile)?;
            csv::WriterBuilder::new()
                .has_headers(true)
                .from_writer(file)
        }
    };
    for result in expanded_results {
        wtr.serialize(result)?;
    }
    wtr.flush()?;

    Ok(())
}

#[cfg(test)]
mod test {
    use std::str::FromStr;

    use chrono::{DateTime, Utc};
    use semver::Version;

    use crate::{
        extract_position_id, extract_shifter_line, extract_version, Position, PositionLayer, PsbId,
    };

    #[test]
    fn parse_position() {
        assert_eq!(
            Position::from_str("A-1-4").unwrap(),
            Position {
                major: PositionLayer::A,
                minor: 1,
                patch: 4
            }
        );
        assert_eq!(
            Position::from_str("A-0-9").unwrap(),
            Position {
                major: PositionLayer::A,
                minor: 0,
                patch: 9
            }
        );
        assert_ne!(
            Position::from_str("A-1-4").unwrap(),
            Position {
                major: PositionLayer::A,
                minor: 0,
                patch: 2
            }
        );
    }

    #[test]
    fn parse_datetime() {
        assert!(DateTime::parse_from_str("2024-06-20T08:42:01+0000", "%+").is_ok());
        assert!(DateTime::parse_from_str("Date: 2024-06-20T08:42:01+0000", "Date: %+").is_ok());
    }

    #[test]
    fn parse_version() {
        assert_eq!(
            extract_version("Shift script: 1.0.0").unwrap(),
            Version::new(1, 0, 0)
        )
    }

    #[test]
    fn parse_shifter_line() {
        assert_eq!(extract_shifter_line("Shifters: foo").unwrap(), "foo");
        assert_eq!(
            extract_shifter_line("Shifters: foo bar").unwrap(),
            "foo bar"
        );
    }

    #[test]
    fn parse_pos_id_line() {
        assert_eq!(
            extract_position_id("Position / assigned-ID : A-0-0 / PS0004").unwrap(),
            (
                Position {
                    major: PositionLayer::A,
                    minor: 0,
                    patch: 0,
                },
                PsbId::new(4)
            )
        );
        assert_eq!(
            extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(),
            (
                Position {
                    major: PositionLayer::A,
                    minor: 1,
                    patch: 7,
                },
                PsbId::new(108)
            )
        );
        assert_ne!(
            extract_position_id("Position / assigned-ID : A-1-7 / PS0108").unwrap(),
            (
                Position {
                    major: PositionLayer::A,
                    minor: 0,
                    patch: 7,
                },
                PsbId::new(106)
            )
        );
    }
}