From 9bf534df1d0add6611774dc0bc6415a7b7885bc1 Mon Sep 17 00:00:00 2001 From: qwjyh Date: Wed, 26 Feb 2025 23:35:49 +0900 Subject: [PATCH 1/7] fix(sync): add pull (only fast forward) --- src/cmd_sync.rs | 235 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 194 insertions(+), 41 deletions(-) diff --git a/src/cmd_sync.rs b/src/cmd_sync.rs index 2c2a213..c462c87 100644 --- a/src/cmd_sync.rs +++ b/src/cmd_sync.rs @@ -1,7 +1,10 @@ -use std::path::{Path, PathBuf}; +use std::{ + io::{self, Write}, + path::{Path, PathBuf}, +}; -use anyhow::{anyhow, Result}; -use git2::{Cred, PushOptions, RemoteCallbacks, Repository}; +use anyhow::{anyhow, Context, Result}; +use git2::{build::CheckoutBuilder, Cred, FetchOptions, PushOptions, RemoteCallbacks, Repository}; pub(crate) fn cmd_sync( config_dir: &PathBuf, @@ -21,12 +24,154 @@ pub(crate) fn cmd_sync( remotes.get(0).unwrap().to_string() } }; + debug!("remote name: {remote_name}"); + let mut remote = repo.find_remote(&remote_name)?; + let callbacks = remote_callback(&use_sshagent, &ssh_key); + let mut fetchoptions = FetchOptions::new(); + fetchoptions.remote_callbacks(callbacks); + let fetch_refspec: Vec = remote + .refspecs() + .filter_map(|rs| match rs.direction() { + git2::Direction::Fetch => rs.str().map(|s| s.to_string()), + git2::Direction::Push => None, + }) + .collect(); + remote + .fetch(&fetch_refspec, Some(&mut fetchoptions), None) + .context("Failed to fetch (pull)")?; + let stats = remote.stats(); + if stats.local_objects() > 0 { + println!( + "\rReceived {}/{} objects in {} bytes (used {} local objects)", + stats.indexed_objects(), + stats.total_objects(), + stats.received_bytes(), + stats.local_objects(), + ); + } else { + println!( + "\rReceived {}/{} objects in {} bytes", + stats.indexed_objects(), + stats.total_objects(), + stats.received_bytes(), + ); + } + let fetch_head = repo + .reference_to_annotated_commit( + &repo + .resolve_reference_from_short_name(&remote_name) + .context("failed to get reference from fetch refspec")?, + ) + .context("failed to get annotated commit")?; + let (merge_analysis, merge_preference) = repo + .merge_analysis(&[&fetch_head]) + .context("failed to do merge_analysis")?; + + trace!("merge analysis: {:?}", merge_analysis); + trace!("merge preference: {:?}", merge_preference); + match merge_analysis { + ma if ma.is_up_to_date() => { + info!("HEAD is up to date. skip merging"); + } + ma if ma.is_fast_forward() => { + // https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs + info!("fast forward is available"); + let mut ref_remote = repo + .find_reference( + remote + .default_branch() + .context("failed to get remote default branch")? + .as_str() + .unwrap(), + ) + .context("failed to get remote reference")?; + let name = match ref_remote.name() { + Some(s) => s.to_string(), + None => String::from_utf8_lossy(ref_remote.name_bytes()).to_string(), + }; + let msg = format!("Fast-Forward: Setting {} to id: {}", name, fetch_head.id()); + println!("{}", msg); + ref_remote + .set_target(fetch_head.id(), &msg) + .context("failed to set target")?; + repo.checkout_head(Some(CheckoutBuilder::default().force())) + .context("failed to checkout")?; + } + ma if ma.is_unborn() => { + warn!("HEAD is invalid (unborn)"); + return Err(anyhow!( + "HEAD is invalid: merge_analysis: {:?}", + merge_analysis + )); + } + ma if ma.is_normal() => { + error!("unable to fast-forward. manual merge is required"); + return Err(anyhow!("unable to fast-forward. manual merge is required")); + } + ma if ma.is_none() => { + error!("no merge is possible"); + return Err(anyhow!("no merge is possible")); + } + _ma => { + error!( + "this code must not reachable: merge_analysis {:?}", + merge_analysis + ); + return Err(anyhow!("must not be reachabel (uncovered merge_analysis)")); + } + } + + // push + let callbacks = remote_callback(&use_sshagent, &ssh_key); + let mut push_options = PushOptions::new(); + push_options.remote_callbacks(callbacks); + trace!("remote: {:?}", remote.name()); + let num_refspecs = remote + .refspecs() + .filter(|rs| rs.direction() == git2::Direction::Push) + .count(); + if num_refspecs > 1 { + warn!("more than one push refspecs are configured"); + warn!("using the first one"); + } + let head = repo.head().context("Failed to get HEAD")?; + if num_refspecs >= 1 { + trace!("using push refspec"); + let push_refspec = remote + .refspecs() + .filter_map(|rs| match rs.direction() { + git2::Direction::Fetch => None, + git2::Direction::Push => Some(rs), + }) + .next() + .expect("this must be unreachabe") + .str() + .context("failed to get valid utf8 push refspec")? + .to_string(); + remote.push(&[push_refspec.as_str()] as &[&str], Some(&mut push_options))?; + } else { + trace!("using head as push refspec"); + trace!("head is branch: {:?}", head.is_branch()); + trace!("head is remote: {:?}", head.is_remote()); + let push_refspec = head.name().context("failed to get head name")?; + remote.push(&[push_refspec] as &[&str], Some(&mut push_options))?; + }; + Ok(()) +} + +fn remote_callback<'b, 'a>( + use_sshagent: &'a bool, + ssh_key: &'b Option, +) -> RemoteCallbacks<'a> +where + 'b: 'a, +{ // using credentials let mut callbacks = RemoteCallbacks::new(); callbacks - .credentials(|_url, username_from_url, _allowed_types| { - if let Some(key) = &ssh_key { + .credentials(move |_url, username_from_url, _allowed_types| { + if let Some(ref key) = ssh_key { info!("Using provided ssh key to access the repository"); let passwd = match inquire::Password::new("SSH passphrase").prompt() { std::result::Result::Ok(s) => Some(s), @@ -42,7 +187,7 @@ pub(crate) fn cmd_sync( key as &Path, passwd.as_deref(), ) - } else if use_sshagent { + } else if *use_sshagent { // use ssh agent info!("Using ssh agent to access the repository"); Cred::ssh_key_from_agent( @@ -54,41 +199,49 @@ pub(crate) fn cmd_sync( panic!("This option must be unreachable.") } }) + .transfer_progress(|progress| { + if progress.received_objects() == progress.total_objects() { + print!( + "Resolving deltas {}/{}\r", + progress.indexed_deltas(), + progress.total_deltas() + ); + } else { + print!( + "Received {}/{} objects ({}) in {} bytes\r", + progress.received_objects(), + progress.total_objects(), + progress.indexed_objects(), + progress.received_bytes(), + ); + } + io::stdout().flush().unwrap(); + true + }) + .sideband_progress(|text| { + let msg = String::from_utf8_lossy(text); + eprintln!("remote: {msg}"); + true + }) .push_transfer_progress(|current, total, bytes| { - trace!("{current},\t{total},\t{bytes}"); + trace!("{current}/{total} files sent \t{bytes} bytes"); + }) + .push_update_reference(|reference_name, status_msg| { + debug!("remote reference_name {reference_name}"); + match status_msg { + None => { + info!("successfully pushed"); + eprintln!("successfully pushed to {}", reference_name); + Ok(()) + } + Some(status) => { + error!("failed to push: {}", status); + Err(git2::Error::from_str(&format!( + "failed to push to {}: {}", + reference_name, status + ))) + } + } }); - callbacks.push_update_reference(|reference_name, status_msg| { - debug!("remote reference_name {reference_name}"); - match status_msg { - None => { - info!("successfully pushed"); - eprintln!("successfully pushed to {}", reference_name); - Ok(()) - } - Some(status) => { - error!("failed to push: {}", status); - Err(git2::Error::from_str(&format!( - "failed to push to {}: {}", - reference_name, status - ))) - } - } - }); - let mut push_options = PushOptions::new(); - push_options.remote_callbacks(callbacks); - let mut remote = repo.find_remote(&remote_name)?; - trace!("remote: {:?}", remote.name()); - if remote.refspecs().len() != 1 { - warn!("multiple refspecs found"); - } - trace!("refspec: {:?}", remote.get_refspec(0).unwrap().str()); - trace!("refspec: {:?}", remote.get_refspec(0).unwrap().direction()); - trace!("refspec: {:?}", repo.head().unwrap().name()); - trace!("head is branch: {:?}", repo.head().unwrap().is_branch()); - trace!("head is remote: {:?}", repo.head().unwrap().is_remote()); - remote.push( - &[repo.head().unwrap().name().unwrap()] as &[&str], - Some(&mut push_options), - )?; - Ok(()) + callbacks } From 99714b4b29a42df79e9a2a9f6bc50baa413b8e8e Mon Sep 17 00:00:00 2001 From: qwjyh Date: Thu, 27 Feb 2025 01:07:44 +0900 Subject: [PATCH 2/7] update(test): use sync command The first sync from 2nd device didn't work, maybe due to that it is the first push. --- tests/cli.rs | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/tests/cli.rs b/tests/cli.rs index 44e93cd..b282410 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2,6 +2,7 @@ mod integrated_test { use std::{ fs::{self, DirBuilder, File}, io::{self, BufWriter, Write}, + path, }; use anyhow::{anyhow, Context, Ok, Result}; @@ -72,6 +73,16 @@ mod integrated_test { Ok(()) } + fn run_sync_cmd(config_dir: &path::Path) -> Result<()> { + Command::cargo_bin("xdbm")? + .arg("-c") + .arg(config_dir) + .args(["sync", "-vvvv"]) + .assert() + .success(); + Ok(()) + } + #[test] fn single_device() -> Result<()> { let config_dir = assert_fs::TempDir::new()?; @@ -380,16 +391,8 @@ mod integrated_test { std::fs::read_to_string(config_dir_1.join("storages.yml"))?.contains("parent: gdrive1") ); - std::process::Command::new("git") - .arg("push") - .current_dir(&config_dir_1) - .assert() - .success(); - std::process::Command::new("git") - .arg("pull") - .current_dir(&config_dir_2) - .assert() - .success(); + run_sync_cmd(&config_dir_1)?; + run_sync_cmd(&config_dir_2)?; // bind // @@ -603,16 +606,8 @@ mod integrated_test { .and(predicate::str::contains("foodoc").not()), ); - std::process::Command::new("git") - .arg("push") - .current_dir(&config_dir_2) - .assert() - .success(); - std::process::Command::new("git") - .arg("pull") - .current_dir(&config_dir_1) - .assert() - .success(); + run_sync_cmd(&config_dir_2)?; + run_sync_cmd(&config_dir_1)?; // bind // @@ -727,16 +722,8 @@ mod integrated_test { .assert() .success(); - std::process::Command::new("git") - .arg("push") - .current_dir(&config_dir_1) - .assert() - .success(); - std::process::Command::new("git") - .arg("pull") - .current_dir(&config_dir_2) - .assert() - .success(); + run_sync_cmd(&config_dir_1)?; + run_sync_cmd(&config_dir_2)?; // backup add // From af9f57b408e0f3bcee1f57a88605c6e0c61f9ac4 Mon Sep 17 00:00:00 2001 From: qwjyh Date: Thu, 27 Feb 2025 02:09:16 +0900 Subject: [PATCH 3/7] refactor: separate push and pull to funcs --- src/cmd_sync.rs | 205 +++++++++++++++++++++++++++--------------------- 1 file changed, 116 insertions(+), 89 deletions(-) diff --git a/src/cmd_sync.rs b/src/cmd_sync.rs index c462c87..377d116 100644 --- a/src/cmd_sync.rs +++ b/src/cmd_sync.rs @@ -27,7 +27,113 @@ pub(crate) fn cmd_sync( debug!("remote name: {remote_name}"); let mut remote = repo.find_remote(&remote_name)?; - let callbacks = remote_callback(&use_sshagent, &ssh_key); + + pull( + &repo, + &mut remote, + remote_name, + &use_sshagent, + ssh_key.as_ref(), + )?; + + push(&repo, &mut remote, &use_sshagent, ssh_key.as_ref())?; + Ok(()) +} + +fn remote_callback<'b, 'a>( + use_sshagent: &'a bool, + ssh_key: Option<&'a PathBuf>, +) -> RemoteCallbacks<'a> +where + 'b: 'a, +{ + // using credentials + let mut callbacks = RemoteCallbacks::new(); + callbacks + .credentials(move |_url, username_from_url, _allowed_types| { + if let Some(key) = ssh_key { + info!("Using provided ssh key to access the repository"); + let passwd = match inquire::Password::new("SSH passphrase").prompt() { + std::result::Result::Ok(s) => Some(s), + Err(err) => { + error!("Failed to get ssh passphrase: {:?}", err); + None + } + }; + Cred::ssh_key( + username_from_url + .ok_or(git2::Error::from_str("No username found from the url"))?, + None, + key as &Path, + passwd.as_deref(), + ) + } else if *use_sshagent { + // use ssh agent + info!("Using ssh agent to access the repository"); + Cred::ssh_key_from_agent( + username_from_url + .ok_or(git2::Error::from_str("No username found from the url"))?, + ) + } else { + error!("no ssh_key and use_sshagent"); + panic!("This option must be unreachable.") + } + }) + .transfer_progress(|progress| { + if progress.received_objects() == progress.total_objects() { + print!( + "Resolving deltas {}/{}\r", + progress.indexed_deltas(), + progress.total_deltas() + ); + } else { + print!( + "Received {}/{} objects ({}) in {} bytes\r", + progress.received_objects(), + progress.total_objects(), + progress.indexed_objects(), + progress.received_bytes(), + ); + } + io::stdout().flush().unwrap(); + true + }) + .sideband_progress(|text| { + let msg = String::from_utf8_lossy(text); + eprintln!("remote: {msg}"); + true + }) + .push_transfer_progress(|current, total, bytes| { + trace!("{current}/{total} files sent \t{bytes} bytes"); + }) + .push_update_reference(|reference_name, status_msg| { + debug!("remote reference_name {reference_name}"); + match status_msg { + None => { + info!("successfully pushed"); + eprintln!("successfully pushed to {}", reference_name); + Ok(()) + } + Some(status) => { + error!("failed to push: {}", status); + Err(git2::Error::from_str(&format!( + "failed to push to {}: {}", + reference_name, status + ))) + } + } + }); + callbacks +} + +fn pull( + repo: &Repository, + remote: &mut git2::Remote, + remote_name: String, + use_sshagent: &bool, + ssh_key: Option<&PathBuf>, +) -> Result<()> { + let callbacks = remote_callback(use_sshagent, ssh_key); let mut fetchoptions = FetchOptions::new(); fetchoptions.remote_callbacks(callbacks); let fetch_refspec: Vec = remote @@ -121,9 +227,16 @@ pub(crate) fn cmd_sync( return Err(anyhow!("must not be reachabel (uncovered merge_analysis)")); } } + Ok(()) +} - // push - let callbacks = remote_callback(&use_sshagent, &ssh_key); +fn push( + repo: &Repository, + remote: &mut git2::Remote, + use_sshagent: &bool, + ssh_key: Option<&PathBuf>, +) -> Result<()> { + let callbacks = remote_callback(&use_sshagent, ssh_key); let mut push_options = PushOptions::new(); push_options.remote_callbacks(callbacks); trace!("remote: {:?}", remote.name()); @@ -159,89 +272,3 @@ pub(crate) fn cmd_sync( }; Ok(()) } - -fn remote_callback<'b, 'a>( - use_sshagent: &'a bool, - ssh_key: &'b Option, -) -> RemoteCallbacks<'a> -where - 'b: 'a, -{ - // using credentials - let mut callbacks = RemoteCallbacks::new(); - callbacks - .credentials(move |_url, username_from_url, _allowed_types| { - if let Some(ref key) = ssh_key { - info!("Using provided ssh key to access the repository"); - let passwd = match inquire::Password::new("SSH passphrase").prompt() { - std::result::Result::Ok(s) => Some(s), - Err(err) => { - error!("Failed to get ssh passphrase: {:?}", err); - None - } - }; - Cred::ssh_key( - username_from_url - .ok_or(git2::Error::from_str("No username found from the url"))?, - None, - key as &Path, - passwd.as_deref(), - ) - } else if *use_sshagent { - // use ssh agent - info!("Using ssh agent to access the repository"); - Cred::ssh_key_from_agent( - username_from_url - .ok_or(git2::Error::from_str("No username found from the url"))?, - ) - } else { - error!("no ssh_key and use_sshagent"); - panic!("This option must be unreachable.") - } - }) - .transfer_progress(|progress| { - if progress.received_objects() == progress.total_objects() { - print!( - "Resolving deltas {}/{}\r", - progress.indexed_deltas(), - progress.total_deltas() - ); - } else { - print!( - "Received {}/{} objects ({}) in {} bytes\r", - progress.received_objects(), - progress.total_objects(), - progress.indexed_objects(), - progress.received_bytes(), - ); - } - io::stdout().flush().unwrap(); - true - }) - .sideband_progress(|text| { - let msg = String::from_utf8_lossy(text); - eprintln!("remote: {msg}"); - true - }) - .push_transfer_progress(|current, total, bytes| { - trace!("{current}/{total} files sent \t{bytes} bytes"); - }) - .push_update_reference(|reference_name, status_msg| { - debug!("remote reference_name {reference_name}"); - match status_msg { - None => { - info!("successfully pushed"); - eprintln!("successfully pushed to {}", reference_name); - Ok(()) - } - Some(status) => { - error!("failed to push: {}", status); - Err(git2::Error::from_str(&format!( - "failed to push to {}: {}", - reference_name, status - ))) - } - } - }); - callbacks -} From 2452a023ade02b4d312010a422e940d4ce3f3994 Mon Sep 17 00:00:00 2001 From: qwjyh Date: Thu, 27 Feb 2025 02:14:41 +0900 Subject: [PATCH 4/7] refactor: change merge analysis match order --- src/cmd_sync.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cmd_sync.rs b/src/cmd_sync.rs index 377d116..4b92d72 100644 --- a/src/cmd_sync.rs +++ b/src/cmd_sync.rs @@ -211,14 +211,14 @@ fn pull( merge_analysis )); } - ma if ma.is_normal() => { - error!("unable to fast-forward. manual merge is required"); - return Err(anyhow!("unable to fast-forward. manual merge is required")); - } ma if ma.is_none() => { error!("no merge is possible"); return Err(anyhow!("no merge is possible")); } + ma if ma.is_normal() => { + error!("unable to fast-forward. manual merge is required"); + return Err(anyhow!("unable to fast-forward. manual merge is required")); + } _ma => { error!( "this code must not reachable: merge_analysis {:?}", From 152e9b2f2d3d68f50072aae1ca484d6bf90c74ca Mon Sep 17 00:00:00 2001 From: qwjyh Date: Thu, 27 Feb 2025 02:30:54 +0900 Subject: [PATCH 5/7] refactor: add logs --- src/cmd_sync.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cmd_sync.rs b/src/cmd_sync.rs index 4b92d72..9611c7a 100644 --- a/src/cmd_sync.rs +++ b/src/cmd_sync.rs @@ -12,7 +12,7 @@ pub(crate) fn cmd_sync( use_sshagent: bool, ssh_key: Option, ) -> Result<()> { - warn!("Experimental"); + info!("cmd_sync"); let repo = Repository::open(config_dir)?; let remote_name = match remote_name { Some(remote_name) => remote_name, @@ -24,7 +24,7 @@ pub(crate) fn cmd_sync( remotes.get(0).unwrap().to_string() } }; - debug!("remote name: {remote_name}"); + debug!("resolved remote name: {remote_name}"); let mut remote = repo.find_remote(&remote_name)?; @@ -133,6 +133,7 @@ fn pull( use_sshagent: &bool, ssh_key: Option<&PathBuf>, ) -> Result<()> { + debug!("pull"); let callbacks = remote_callback(use_sshagent, ssh_key); let mut fetchoptions = FetchOptions::new(); fetchoptions.remote_callbacks(callbacks); @@ -236,20 +237,20 @@ fn push( use_sshagent: &bool, ssh_key: Option<&PathBuf>, ) -> Result<()> { + debug!("push"); let callbacks = remote_callback(&use_sshagent, ssh_key); let mut push_options = PushOptions::new(); push_options.remote_callbacks(callbacks); - trace!("remote: {:?}", remote.name()); - let num_refspecs = remote + let num_push_refspecs = remote .refspecs() .filter(|rs| rs.direction() == git2::Direction::Push) .count(); - if num_refspecs > 1 { + if num_push_refspecs > 1 { warn!("more than one push refspecs are configured"); warn!("using the first one"); } let head = repo.head().context("Failed to get HEAD")?; - if num_refspecs >= 1 { + if num_push_refspecs >= 1 { trace!("using push refspec"); let push_refspec = remote .refspecs() From c08d455780dada59bf4fe563273ebb7303c79c6d Mon Sep 17 00:00:00 2001 From: qwjyh Date: Thu, 27 Feb 2025 02:38:07 +0900 Subject: [PATCH 6/7] update: CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a1a66f..1429919 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Added - Add `status` subcommand to see storage and backup on given path or current working directory ([#17](https://github.com/qwjyh/xdbm/pull/17)). +- `sync` subcommand, which performs git pull (fast-forward) and push ### Changed - Colored output for `storage list` and `backup list` ([#15](https://github.com/qwjyh/xdbm/pull/15)) From c2b6506db0b3326715ac8e1d27c32a851c04ae0a Mon Sep 17 00:00:00 2001 From: qwjyh Date: Thu, 27 Feb 2025 02:48:03 +0900 Subject: [PATCH 7/7] fix: CHANGELOG was wrong --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1429919..575d09b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Added +- `sync` subcommand, which performs git pull (fast-forward) and push (#21) + ### Fixed - Git local config is now looked up. (#20) - Git global config will not be polluted in test by default. (#20) @@ -10,7 +13,6 @@ ### Added - Add `status` subcommand to see storage and backup on given path or current working directory ([#17](https://github.com/qwjyh/xdbm/pull/17)). -- `sync` subcommand, which performs git pull (fast-forward) and push ### Changed - Colored output for `storage list` and `backup list` ([#15](https://github.com/qwjyh/xdbm/pull/15))