implement sync subcommand (#21)

* new(sync): implement sync subcommand (WIP)

TODO
- update CHANGELOG
- refactor sync func

* fix(sync): add pull (only fast forward)

* update(test): use sync command

The first sync from 2nd device didn't work, maybe due to that it is the
first push.

* refactor: separate push and pull to funcs

* refactor: change merge analysis match order

* refactor: add logs

* update: CHANGELOG

* fix: CHANGELOG was wrong

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## [Unreleased]
 
+### Added
+- `sync` subcommand, which performs git pull (fast-forward) and push (#21)
+
 ### Fixed
 - Git local config is now looked up. (#20)
 - Git global config will not be polluted in test by default. (#20)

src/cmd_args.rs

@@ -63,6 +63,12 @@ pub(crate) enum Commands {
     Sync {
         /// Remote name to sync.
         remote_name: Option<String>,
+        /// Whether to use ssh-agent
+        #[arg(long)]
+        use_sshagent: bool,
+        /// Manually specify ssh key
+        #[arg(long)]
+        ssh_key: Option<PathBuf>,
     },
 
     /// Check config files validity.

src/cmd_init.rs

@@ -40,9 +40,7 @@ fn clone_repo(
                 }
             };
             Cred::ssh_key(
-                username_from_url
-                    .context("No username found from the url")
-                    .unwrap(),
+                username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?,
                 None,
                 key as &Path,
                 passwd.as_deref(),
@@ -51,9 +49,7 @@ fn clone_repo(
             // use ssh agent
             info!("Using ssh agent to access the repository");
             Cred::ssh_key_from_agent(
-                username_from_url
-                    .context("No username found from the url")
-                    .unwrap(),
+                username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?,
             )
         } else {
             error!("no ssh_key and use_sshagent");

src/cmd_sync.rs

@@ -1,10 +1,18 @@
-use std::path::PathBuf;
+use std::{
+    io::{self, Write},
+    path::{Path, PathBuf},
+};
 
-use anyhow::{anyhow, Result};
-use git2::Repository;
+use anyhow::{anyhow, Context, Result};
+use git2::{build::CheckoutBuilder, Cred, FetchOptions, PushOptions, RemoteCallbacks, Repository};
 
-pub(crate) fn cmd_sync(config_dir: &PathBuf, remote_name: Option<String>) -> Result<()> {
-    warn!("Experimental");
+pub(crate) fn cmd_sync(
+    config_dir: &PathBuf,
+    remote_name: Option<String>,
+    use_sshagent: bool,
+    ssh_key: Option<PathBuf>,
+) -> Result<()> {
+    info!("cmd_sync");
     let repo = Repository::open(config_dir)?;
     let remote_name = match remote_name {
         Some(remote_name) => remote_name,
@@ -16,7 +24,252 @@ pub(crate) fn cmd_sync(config_dir: &PathBuf, remote_name: Option<String>) -> Res
             remotes.get(0).unwrap().to_string()
         }
     };
+    debug!("resolved remote name: {remote_name}");
+
     let mut remote = repo.find_remote(&remote_name)?;
-    remote.push(&[] as &[&str], None)?;
+
+    pull(
+        &repo,
+        &mut remote,
+        remote_name,
+        &use_sshagent,
+        ssh_key.as_ref(),
+    )?;
+
+    push(&repo, &mut remote, &use_sshagent, ssh_key.as_ref())?;
+    Ok(())
+}
+
+fn remote_callback<'b, 'a>(
+    use_sshagent: &'a bool,
+    ssh_key: Option<&'a PathBuf>,
+) -> RemoteCallbacks<'a>
+where
+    'b: 'a,
+{
+    // using credentials
+    let mut callbacks = RemoteCallbacks::new();
+    callbacks
+        .credentials(move |_url, username_from_url, _allowed_types| {
+            if let Some(key) = ssh_key {
+                info!("Using provided ssh key to access the repository");
+                let passwd = match inquire::Password::new("SSH passphrase").prompt() {
+                    std::result::Result::Ok(s) => Some(s),
+                    Err(err) => {
+                        error!("Failed to get ssh passphrase: {:?}", err);
+                        None
+                    }
+                };
+                Cred::ssh_key(
+                    username_from_url
+                        .ok_or(git2::Error::from_str("No username found from the url"))?,
+                    None,
+                    key as &Path,
+                    passwd.as_deref(),
+                )
+            } else if *use_sshagent {
+                // use ssh agent
+                info!("Using ssh agent to access the repository");
+                Cred::ssh_key_from_agent(
+                    username_from_url
+                        .ok_or(git2::Error::from_str("No username found from the url"))?,
+                )
+            } else {
+                error!("no ssh_key and use_sshagent");
+                panic!("This option must be unreachable.")
+            }
+        })
+        .transfer_progress(|progress| {
+            if progress.received_objects() == progress.total_objects() {
+                print!(
+                    "Resolving deltas {}/{}\r",
+                    progress.indexed_deltas(),
+                    progress.total_deltas()
+                );
+            } else {
+                print!(
+                    "Received {}/{} objects ({}) in {} bytes\r",
+                    progress.received_objects(),
+                    progress.total_objects(),
+                    progress.indexed_objects(),
+                    progress.received_bytes(),
+                );
+            }
+            io::stdout().flush().unwrap();
+            true
+        })
+        .sideband_progress(|text| {
+            let msg = String::from_utf8_lossy(text);
+            eprintln!("remote: {msg}");
+            true
+        })
+        .push_transfer_progress(|current, total, bytes| {
+            trace!("{current}/{total} files sent \t{bytes} bytes");
+        })
+        .push_update_reference(|reference_name, status_msg| {
+            debug!("remote reference_name {reference_name}");
+            match status_msg {
+                None => {
+                    info!("successfully pushed");
+                    eprintln!("successfully pushed to {}", reference_name);
+                    Ok(())
+                }
+                Some(status) => {
+                    error!("failed to push: {}", status);
+                    Err(git2::Error::from_str(&format!(
+                        "failed to push to {}: {}",
+                        reference_name, status
+                    )))
+                }
+            }
+        });
+    callbacks
+}
+
+fn pull(
+    repo: &Repository,
+    remote: &mut git2::Remote,
+    remote_name: String,
+    use_sshagent: &bool,
+    ssh_key: Option<&PathBuf>,
+) -> Result<()> {
+    debug!("pull");
+    let callbacks = remote_callback(use_sshagent, ssh_key);
+    let mut fetchoptions = FetchOptions::new();
+    fetchoptions.remote_callbacks(callbacks);
+    let fetch_refspec: Vec<String> = remote
+        .refspecs()
+        .filter_map(|rs| match rs.direction() {
+            git2::Direction::Fetch => rs.str().map(|s| s.to_string()),
+            git2::Direction::Push => None,
+        })
+        .collect();
+    remote
+        .fetch(&fetch_refspec, Some(&mut fetchoptions), None)
+        .context("Failed to fetch (pull)")?;
+    let stats = remote.stats();
+    if stats.local_objects() > 0 {
+        println!(
+            "\rReceived {}/{} objects in {} bytes (used {} local objects)",
+            stats.indexed_objects(),
+            stats.total_objects(),
+            stats.received_bytes(),
+            stats.local_objects(),
+        );
+    } else {
+        println!(
+            "\rReceived {}/{} objects in {} bytes",
+            stats.indexed_objects(),
+            stats.total_objects(),
+            stats.received_bytes(),
+        );
+    }
+    let fetch_head = repo
+        .reference_to_annotated_commit(
+            &repo
+                .resolve_reference_from_short_name(&remote_name)
+                .context("failed to get reference from fetch refspec")?,
+        )
+        .context("failed to get annotated commit")?;
+    let (merge_analysis, merge_preference) = repo
+        .merge_analysis(&[&fetch_head])
+        .context("failed to do merge_analysis")?;
+
+    trace!("merge analysis: {:?}", merge_analysis);
+    trace!("merge preference: {:?}", merge_preference);
+    match merge_analysis {
+        ma if ma.is_up_to_date() => {
+            info!("HEAD is up to date. skip merging");
+        }
+        ma if ma.is_fast_forward() => {
+            // https://github.com/rust-lang/git2-rs/blob/master/examples/pull.rs
+            info!("fast forward is available");
+            let mut ref_remote = repo
+                .find_reference(
+                    remote
+                        .default_branch()
+                        .context("failed to get remote default branch")?
+                        .as_str()
+                        .unwrap(),
+                )
+                .context("failed to get remote reference")?;
+            let name = match ref_remote.name() {
+                Some(s) => s.to_string(),
+                None => String::from_utf8_lossy(ref_remote.name_bytes()).to_string(),
+            };
+            let msg = format!("Fast-Forward: Setting {} to id: {}", name, fetch_head.id());
+            println!("{}", msg);
+            ref_remote
+                .set_target(fetch_head.id(), &msg)
+                .context("failed to set target")?;
+            repo.checkout_head(Some(CheckoutBuilder::default().force()))
+                .context("failed to checkout")?;
+        }
+        ma if ma.is_unborn() => {
+            warn!("HEAD is invalid (unborn)");
+            return Err(anyhow!(
+                "HEAD is invalid: merge_analysis: {:?}",
+                merge_analysis
+            ));
+        }
+        ma if ma.is_none() => {
+            error!("no merge is possible");
+            return Err(anyhow!("no merge is possible"));
+        }
+        ma if ma.is_normal() => {
+            error!("unable to fast-forward. manual merge is required");
+            return Err(anyhow!("unable to fast-forward. manual merge is required"));
+        }
+        _ma => {
+            error!(
+                "this code must not reachable: merge_analysis {:?}",
+                merge_analysis
+            );
+            return Err(anyhow!("must not be reachabel (uncovered merge_analysis)"));
+        }
+    }
+    Ok(())
+}
+
+fn push(
+    repo: &Repository,
+    remote: &mut git2::Remote,
+    use_sshagent: &bool,
+    ssh_key: Option<&PathBuf>,
+) -> Result<()> {
+    debug!("push");
+    let callbacks = remote_callback(&use_sshagent, ssh_key);
+    let mut push_options = PushOptions::new();
+    push_options.remote_callbacks(callbacks);
+    let num_push_refspecs = remote
+        .refspecs()
+        .filter(|rs| rs.direction() == git2::Direction::Push)
+        .count();
+    if num_push_refspecs > 1 {
+        warn!("more than one push refspecs are configured");
+        warn!("using the first one");
+    }
+    let head = repo.head().context("Failed to get HEAD")?;
+    if num_push_refspecs >= 1 {
+        trace!("using push refspec");
+        let push_refspec = remote
+            .refspecs()
+            .filter_map(|rs| match rs.direction() {
+                git2::Direction::Fetch => None,
+                git2::Direction::Push => Some(rs),
+            })
+            .next()
+            .expect("this must be unreachabe")
+            .str()
+            .context("failed to get valid utf8 push refspec")?
+            .to_string();
+        remote.push(&[push_refspec.as_str()] as &[&str], Some(&mut push_options))?;
+    } else {
+        trace!("using head as push refspec");
+        trace!("head is branch: {:?}", head.is_branch());
+        trace!("head is remote: {:?}", head.is_remote());
+        let push_refspec = head.name().context("failed to get head name")?;
+        remote.push(&[push_refspec] as &[&str], Some(&mut push_options))?;
+    };
     Ok(())
 }

src/git.rs

@@ -0,0 +1,40 @@
+use std::path::{Path, PathBuf};
+
+use git2::{Cred, RemoteCallbacks};
+use inquire::Password;
+
+pub(crate) fn get_credential<'a>(
+    use_sshagent: bool,
+    ssh_key: Option<PathBuf>,
+) -> RemoteCallbacks<'a> {
+    // using credentials
+    let mut callbacks = RemoteCallbacks::new();
+    callbacks.credentials(move |_url, username_from_url, _allowed_types| {
+        if let Some(key) = &ssh_key {
+            info!("Using provided ssh key to access the repository");
+            let passwd = match Password::new("SSH passphrase").prompt() {
+                std::result::Result::Ok(s) => Some(s),
+                Err(err) => {
+                    error!("Failed to get ssh passphrase: {:?}", err);
+                    None
+                }
+            };
+            Cred::ssh_key(
+                username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?,
+                None,
+                key as &Path,
+                passwd.as_deref(),
+            )
+        } else if use_sshagent {
+            // use ssh agent
+            info!("Using ssh agent to access the repository");
+            Cred::ssh_key_from_agent(
+                username_from_url.ok_or(git2::Error::from_str("No username found from the url"))?,
+            )
+        } else {
+            error!("no ssh_key and use_sshagent");
+            panic!("This option must be unreachable.")
+        }
+    });
+    callbacks
+}

src/main.rs

@@ -35,6 +35,7 @@ mod cmd_status;
 mod cmd_storage;
 mod cmd_sync;
 mod devices;
+mod git;
 mod inquire_filepath_completer;
 mod storages;
 mod util;
@@ -91,7 +92,11 @@ fn main() -> Result<()> {
         Commands::Path {} => {
             println!("{}", &config_dir.display());
         }
-        Commands::Sync { remote_name } => cmd_sync::cmd_sync(&config_dir, remote_name)?,
+        Commands::Sync {
+            remote_name,
+            use_sshagent,
+            ssh_key,
+        } => cmd_sync::cmd_sync(&config_dir, remote_name, use_sshagent, ssh_key)?,
         Commands::Status {
             path,
             storage,

tests/cli.rs

@@ -2,6 +2,7 @@ mod integrated_test {
     use std::{
         fs::{self, DirBuilder, File},
         io::{self, BufWriter, Write},
+        path,
     };
 
     use anyhow::{anyhow, Context, Ok, Result};
@@ -72,6 +73,16 @@ mod integrated_test {
         Ok(())
     }
 
+    fn run_sync_cmd(config_dir: &path::Path) -> Result<()> {
+        Command::cargo_bin("xdbm")?
+            .arg("-c")
+            .arg(config_dir)
+            .args(["sync", "-vvvv"])
+            .assert()
+            .success();
+        Ok(())
+    }
+
     #[test]
     fn single_device() -> Result<()> {
         let config_dir = assert_fs::TempDir::new()?;
@@ -313,15 +324,14 @@ mod integrated_test {
         assert!(config_dir_2.join("backups").join("second.yml").exists());
 
         // sync
-        std::process::Command::new("git")
-            .arg("push")
-            .current_dir(&config_dir_2)
+        Command::cargo_bin("xdbm")?
+            .arg("-c")
+            .arg(config_dir_2.path())
+            .arg("sync")
+            .arg("-vvvv")
             .assert()
-            .success();
-        // let repo_2 = Repository::open(config_dir_2)?;
-        // // return Err(anyhow!("{:?}", repo_2.remotes()?.iter().collect::<Vec<_>>()));
-        // let mut repo_2_remote = repo_2.find_remote(repo_2.remotes()?.get(0).unwrap())?;
-        // repo_2_remote.push(&[] as &[&str], None)?;
+            .success()
+            .stderr(predicate::str::contains("successfully pushed"));
         std::process::Command::new("git")
             .arg("pull")
             .current_dir(&config_dir_1)
@@ -381,16 +391,8 @@ mod integrated_test {
             std::fs::read_to_string(config_dir_1.join("storages.yml"))?.contains("parent: gdrive1")
         );
 
-        std::process::Command::new("git")
-            .arg("push")
-            .current_dir(&config_dir_1)
-            .assert()
-            .success();
-        std::process::Command::new("git")
-            .arg("pull")
-            .current_dir(&config_dir_2)
-            .assert()
-            .success();
+        run_sync_cmd(&config_dir_1)?;
+        run_sync_cmd(&config_dir_2)?;
 
         // bind
         //
@@ -604,16 +606,8 @@ mod integrated_test {
                     .and(predicate::str::contains("foodoc").not()),
             );
 
-        std::process::Command::new("git")
-            .arg("push")
-            .current_dir(&config_dir_2)
-            .assert()
-            .success();
-        std::process::Command::new("git")
-            .arg("pull")
-            .current_dir(&config_dir_1)
-            .assert()
-            .success();
+        run_sync_cmd(&config_dir_2)?;
+        run_sync_cmd(&config_dir_1)?;
 
         // bind
         //
@@ -728,16 +722,8 @@ mod integrated_test {
             .assert()
             .success();
 
-        std::process::Command::new("git")
-            .arg("push")
-            .current_dir(&config_dir_1)
-            .assert()
-            .success();
-        std::process::Command::new("git")
-            .arg("pull")
-            .current_dir(&config_dir_2)
-            .assert()
-            .success();
+        run_sync_cmd(&config_dir_1)?;
+        run_sync_cmd(&config_dir_2)?;
 
         // backup add
         //