From bd8e2019febebbbddedd6727bb6bdcdc7e729ac5 Mon Sep 17 00:00:00 2001 From: qwjyh <62229267+qwjyh@users.noreply.github.com> Date: Thu, 27 Feb 2025 02:56:39 +0900 Subject: [PATCH] implement sync subcommand (#21) * new(sync): implement sync subcommand (WIP) TODO - update CHANGELOG - refactor sync func * fix(sync): add pull (only fast forward) * update(test): use sync command The first sync from 2nd device didn't work, maybe due to that it is the first push. * refactor: separate push and pull to funcs * refactor: change merge analysis match order * refactor: add logs * update: CHANGELOG * fix: CHANGELOG was wrong --- CHANGELOG.md | 3 + src/cmd_args.rs | 6 ++ src/cmd_init.rs | 8 +- src/cmd_sync.rs | 265 ++++++++++++++++++++++++++++++++++++++++++++++-- src/git.rs | 40 ++++++++ src/main.rs | 7 +- tests/cli.rs | 62 +++++------ 7 files changed, 340 insertions(+), 51 deletions(-) create mode 100644 src/git.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a1a66f..575d09b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Added +- `sync` subcommand, which performs git pull (fast-forward) and push (#21) + ### Fixed - Git local config is now looked up. (#20) - Git global config will not be polluted in test by default. (#20) diff --git a/src/cmd_args.rs b/src/cmd_args.rs index c0b46b7..1c3eb3c 100644 --- a/src/cmd_args.rs +++ b/src/cmd_args.rs @@ -63,6 +63,12 @@ pub(crate) enum Commands { Sync { /// Remote name to sync. remote_name: Option, + /// Whether to use ssh-agent + #[arg(long)] + use_sshagent: bool, + /// Manually specify ssh key + #[arg(long)] + ssh_key: Option, }, /// Check config files validity. diff --git a/src/cmd_init.rs b/src/cmd_init.rs index 415489a..7207202 100644 --- a/src/cmd_init.rs +++ b/src/cmd_init.rs @@ -40,9 +40,7 @@ fn clone_repo( } }; Cred::ssh_key( - username_from_url - .context("No username found from the url") - .unwrap(), + username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?, None, key as &Path, passwd.as_deref(), @@ -51,9 +49,7 @@ fn clone_repo( // use ssh agent info!("Using ssh agent to access the repository"); Cred::ssh_key_from_agent( - username_from_url - .context("No username found from the url") - .unwrap(), + username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?, ) } else { error!("no ssh_key and use_sshagent"); diff --git a/src/cmd_sync.rs b/src/cmd_sync.rs index eb477ca..9611c7a 100644 --- a/src/cmd_sync.rs +++ b/src/cmd_sync.rs @@ -1,10 +1,18 @@ -use std::path::PathBuf; +use std::{ + io::{self, Write}, + path::{Path, PathBuf}, +}; -use anyhow::{anyhow, Result}; -use git2::Repository; +use anyhow::{anyhow, Context, Result}; +use git2::{build::CheckoutBuilder, Cred, FetchOptions, PushOptions, RemoteCallbacks, Repository}; -pub(crate) fn cmd_sync(config_dir: &PathBuf, remote_name: Option) -> Result<()> { - warn!("Experimental"); +pub(crate) fn cmd_sync( + config_dir: &PathBuf, + remote_name: Option, + use_sshagent: bool, + ssh_key: Option, +) -> Result<()> { + info!("cmd_sync"); let repo = Repository::open(config_dir)?; let remote_name = match remote_name { Some(remote_name) => remote_name, @@ -16,7 +24,252 @@ pub(crate) fn cmd_sync(config_dir: &PathBuf, remote_name: Option) -> Res remotes.get(0).unwrap().to_string() } }; + debug!("resolved remote name: {remote_name}"); + let mut remote = repo.find_remote(&remote_name)?; - remote.push(&[] as &[&str], None)?; + + pull( + &repo, + &mut remote, + remote_name, + &use_sshagent, + ssh_key.as_ref(), + )?; + + push(&repo, &mut remote, &use_sshagent, ssh_key.as_ref())?; + Ok(()) +} + +fn remote_callback<'b, 'a>( + use_sshagent: &'a bool, + ssh_key: Option<&'a PathBuf>, +) -> RemoteCallbacks<'a> +where + 'b: 'a, +{ + // using credentials + let mut callbacks = RemoteCallbacks::new(); + callbacks + .credentials(move |_url, username_from_url, _allowed_types| { + if let Some(key) = ssh_key { + info!("Using provided ssh key to access the repository"); + let passwd = match inquire::Password::new("SSH passphrase").prompt() { + std::result::Result::Ok(s) => Some(s), + Err(err) => { + error!("Failed to get ssh passphrase: {:?}", err); + None + } + }; + Cred::ssh_key( + username_from_url + .ok_or(git2::Error::from_str("No username found from the url"))?, + None, + key as &Path, + passwd.as_deref(), + ) + } else if *use_sshagent { + // use ssh agent + info!("Using ssh agent to access the repository"); + Cred::ssh_key_from_agent( + username_from_url + .ok_or(git2::Error::from_str("No username found from the url"))?, + ) + } else { + error!("no ssh_key and use_sshagent"); + panic!("This option must be unreachable.") + } + }) + .transfer_progress(|progress| { + if progress.received_objects() == progress.total_objects() { + print!( + "Resolving deltas {}/{}\r", + progress.indexed_deltas(), + progress.total_deltas() + ); + } else { + print!( + "Received {}/{} objects ({}) in {} bytes\r", + progress.received_objects(), + progress.total_objects(), + progress.indexed_objects(), + progress.received_bytes(), + ); + } + io::stdout().flush().unwrap(); + true + }) + .sideband_progress(|text| { + let msg = String::from_utf8_lossy(text); + eprintln!("remote: {msg}"); + true + }) + .push_transfer_progress(|current, total, bytes| { + trace!("{current}/{total} files sent \t{bytes} bytes"); + }) + .push_update_reference(|reference_name, status_msg| { + debug!("remote reference_name {reference_name}"); + match status_msg { + None => { + info!("successfully pushed"); + eprintln!("successfully pushed to {}", reference_name); + Ok(()) + } + Some(status) => { + error!("failed to push: {}", status); + Err(git2::Error::from_str(&format!( + "failed to push to {}: {}", + reference_name, status + ))) + } + } + }); + callbacks +} + +fn pull( + repo: &Repository, + remote: &mut git2::Remote, + remote_name: String, + use_sshagent: &bool, + ssh_key: Option<&PathBuf>, +) -> Result<()> { + debug!("pull"); + let callbacks = remote_callback(use_sshagent, ssh_key); + let mut fetchoptions = FetchOptions::new(); + fetchoptions.remote_callbacks(callbacks); + let fetch_refspec: Vec = remote + .refspecs() + .filter_map(|rs| match rs.direction() { + git2::Direction::Fetch => rs.str().map(|s| s.to_string()), + git2::Direction::Push => None, + }) + .collect(); + remote + .fetch(&fetch_refspec, Some(&mut fetchoptions), None) + .context("Failed to fetch (pull)")?; + let stats = remote.stats(); + if stats.local_objects() > 0 { + println!( + "\rReceived {}/{} objects in {} bytes (used {} local objects)", + stats.indexed_objects(), + stats.total_objects(), + stats.received_bytes(), + stats.local_objects(), + ); + } else { + println!( + "\rReceived {}/{} objects in {} bytes", + stats.indexed_objects(), + stats.total_objects(), + stats.received_bytes(), + ); + } + let fetch_head = repo + .reference_to_annotated_commit( + &repo + .resolve_reference_from_short_name(&remote_name) + .context("failed to get reference from fetch refspec")?, + ) + .context("failed to get annotated commit")?; + let (merge_analysis, merge_preference) = repo + .merge_analysis(&[&fetch_head]) + .context("failed to do merge_analysis")?; + + trace!("merge analysis: {:?}", merge_analysis); + trace!("merge preference: {:?}", merge_preference); + match merge_analysis { + ma if ma.is_up_to_date() => { + info!("HEAD is up to date. skip merging"); + } + ma if ma.is_fast_forward() => { + // https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs + info!("fast forward is available"); + let mut ref_remote = repo + .find_reference( + remote + .default_branch() + .context("failed to get remote default branch")? + .as_str() + .unwrap(), + ) + .context("failed to get remote reference")?; + let name = match ref_remote.name() { + Some(s) => s.to_string(), + None => String::from_utf8_lossy(ref_remote.name_bytes()).to_string(), + }; + let msg = format!("Fast-Forward: Setting {} to id: {}", name, fetch_head.id()); + println!("{}", msg); + ref_remote + .set_target(fetch_head.id(), &msg) + .context("failed to set target")?; + repo.checkout_head(Some(CheckoutBuilder::default().force())) + .context("failed to checkout")?; + } + ma if ma.is_unborn() => { + warn!("HEAD is invalid (unborn)"); + return Err(anyhow!( + "HEAD is invalid: merge_analysis: {:?}", + merge_analysis + )); + } + ma if ma.is_none() => { + error!("no merge is possible"); + return Err(anyhow!("no merge is possible")); + } + ma if ma.is_normal() => { + error!("unable to fast-forward. manual merge is required"); + return Err(anyhow!("unable to fast-forward. manual merge is required")); + } + _ma => { + error!( + "this code must not reachable: merge_analysis {:?}", + merge_analysis + ); + return Err(anyhow!("must not be reachabel (uncovered merge_analysis)")); + } + } + Ok(()) +} + +fn push( + repo: &Repository, + remote: &mut git2::Remote, + use_sshagent: &bool, + ssh_key: Option<&PathBuf>, +) -> Result<()> { + debug!("push"); + let callbacks = remote_callback(&use_sshagent, ssh_key); + let mut push_options = PushOptions::new(); + push_options.remote_callbacks(callbacks); + let num_push_refspecs = remote + .refspecs() + .filter(|rs| rs.direction() == git2::Direction::Push) + .count(); + if num_push_refspecs > 1 { + warn!("more than one push refspecs are configured"); + warn!("using the first one"); + } + let head = repo.head().context("Failed to get HEAD")?; + if num_push_refspecs >= 1 { + trace!("using push refspec"); + let push_refspec = remote + .refspecs() + .filter_map(|rs| match rs.direction() { + git2::Direction::Fetch => None, + git2::Direction::Push => Some(rs), + }) + .next() + .expect("this must be unreachabe") + .str() + .context("failed to get valid utf8 push refspec")? + .to_string(); + remote.push(&[push_refspec.as_str()] as &[&str], Some(&mut push_options))?; + } else { + trace!("using head as push refspec"); + trace!("head is branch: {:?}", head.is_branch()); + trace!("head is remote: {:?}", head.is_remote()); + let push_refspec = head.name().context("failed to get head name")?; + remote.push(&[push_refspec] as &[&str], Some(&mut push_options))?; + }; Ok(()) } diff --git a/src/git.rs b/src/git.rs new file mode 100644 index 0000000..4ff8970 --- /dev/null +++ b/src/git.rs @@ -0,0 +1,40 @@ +use std::path::{Path, PathBuf}; + +use git2::{Cred, RemoteCallbacks}; +use inquire::Password; + +pub(crate) fn get_credential<'a>( + use_sshagent: bool, + ssh_key: Option, +) -> RemoteCallbacks<'a> { + // using credentials + let mut callbacks = RemoteCallbacks::new(); + callbacks.credentials(move |_url, username_from_url, _allowed_types| { + if let Some(key) = &ssh_key { + info!("Using provided ssh key to access the repository"); + let passwd = match Password::new("SSH passphrase").prompt() { + std::result::Result::Ok(s) => Some(s), + Err(err) => { + error!("Failed to get ssh passphrase: {:?}", err); + None + } + }; + Cred::ssh_key( + username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?, + None, + key as &Path, + passwd.as_deref(), + ) + } else if use_sshagent { + // use ssh agent + info!("Using ssh agent to access the repository"); + Cred::ssh_key_from_agent( + username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?, + ) + } else { + error!("no ssh_key and use_sshagent"); + panic!("This option must be unreachable.") + } + }); + callbacks +} diff --git a/src/main.rs b/src/main.rs index 32cf7d6..41aab33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,7 @@ mod cmd_status; mod cmd_storage; mod cmd_sync; mod devices; +mod git; mod inquire_filepath_completer; mod storages; mod util; @@ -91,7 +92,11 @@ fn main() -> Result<()> { Commands::Path {} => { println!("{}", &config_dir.display()); } - Commands::Sync { remote_name } => cmd_sync::cmd_sync(&config_dir, remote_name)?, + Commands::Sync { + remote_name, + use_sshagent, + ssh_key, + } => cmd_sync::cmd_sync(&config_dir, remote_name, use_sshagent, ssh_key)?, Commands::Status { path, storage, diff --git a/tests/cli.rs b/tests/cli.rs index 06e3890..b282410 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2,6 +2,7 @@ mod integrated_test { use std::{ fs::{self, DirBuilder, File}, io::{self, BufWriter, Write}, + path, }; use anyhow::{anyhow, Context, Ok, Result}; @@ -72,6 +73,16 @@ mod integrated_test { Ok(()) } + fn run_sync_cmd(config_dir: &path::Path) -> Result<()> { + Command::cargo_bin("xdbm")? + .arg("-c") + .arg(config_dir) + .args(["sync", "-vvvv"]) + .assert() + .success(); + Ok(()) + } + #[test] fn single_device() -> Result<()> { let config_dir = assert_fs::TempDir::new()?; @@ -313,15 +324,14 @@ mod integrated_test { assert!(config_dir_2.join("backups").join("second.yml").exists()); // sync - std::process::Command::new("git") - .arg("push") - .current_dir(&config_dir_2) + Command::cargo_bin("xdbm")? + .arg("-c") + .arg(config_dir_2.path()) + .arg("sync") + .arg("-vvvv") .assert() - .success(); - // let repo_2 = Repository::open(config_dir_2)?; - // // return Err(anyhow!("{:?}", repo_2.remotes()?.iter().collect::>())); - // let mut repo_2_remote = repo_2.find_remote(repo_2.remotes()?.get(0).unwrap())?; - // repo_2_remote.push(&[] as &[&str], None)?; + .success() + .stderr(predicate::str::contains("successfully pushed")); std::process::Command::new("git") .arg("pull") .current_dir(&config_dir_1) @@ -381,16 +391,8 @@ mod integrated_test { std::fs::read_to_string(config_dir_1.join("storages.yml"))?.contains("parent: gdrive1") ); - std::process::Command::new("git") - .arg("push") - .current_dir(&config_dir_1) - .assert() - .success(); - std::process::Command::new("git") - .arg("pull") - .current_dir(&config_dir_2) - .assert() - .success(); + run_sync_cmd(&config_dir_1)?; + run_sync_cmd(&config_dir_2)?; // bind // @@ -604,16 +606,8 @@ mod integrated_test { .and(predicate::str::contains("foodoc").not()), ); - std::process::Command::new("git") - .arg("push") - .current_dir(&config_dir_2) - .assert() - .success(); - std::process::Command::new("git") - .arg("pull") - .current_dir(&config_dir_1) - .assert() - .success(); + run_sync_cmd(&config_dir_2)?; + run_sync_cmd(&config_dir_1)?; // bind // @@ -728,16 +722,8 @@ mod integrated_test { .assert() .success(); - std::process::Command::new("git") - .arg("push") - .current_dir(&config_dir_1) - .assert() - .success(); - std::process::Command::new("git") - .arg("pull") - .current_dir(&config_dir_2) - .assert() - .success(); + run_sync_cmd(&config_dir_1)?; + run_sync_cmd(&config_dir_2)?; // backup add //