From bb67b7c66dfc52b16632834935d9b8157c63e1af Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 27 Apr 2024 15:23:42 +0100 Subject: [PATCH] feat(gitforge): clone repo in-process Use the `gix` crate directly to create the clone rather then spawning a `git` processess. Closes kemitix/git-next#54 Closes kemitix/git-next#70 --- Cargo.toml | 8 +++- src/server/config/mod.rs | 14 ++++++- src/server/gitforge/errors.rs | 14 +++++++ src/server/gitforge/forgejo/mod.rs | 7 +++- src/server/gitforge/forgejo/repo/clone.rs | 42 +++++-------------- src/server/mod.rs | 51 ++++++++++++++++++++--- 6 files changed, 94 insertions(+), 42 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aa12aa97..8a1ac396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,13 @@ tracing-subscriber = "0.3" base64 = "0.22" # git -gix = "0.62" +# gix = "0.62" +gix = { version = "0.62", features = [ + "basic", + "extras", + "comfort", + "blocking-http-transport-reqwest-rust-tls", +] } async-trait = "0.1" # fs/network diff --git a/src/server/config/mod.rs b/src/server/config/mod.rs index e433fc81..c98321bd 100644 --- a/src/server/config/mod.rs +++ b/src/server/config/mod.rs @@ -4,7 +4,7 @@ use std::{ collections::HashMap, fmt::{Display, Formatter}, ops::Deref, - path::PathBuf, + path::{Path, PathBuf}, }; use serde::Deserialize; @@ -71,6 +71,11 @@ impl AsRef for WebhookUrl { pub struct ServerStorage { path: PathBuf, } +impl ServerStorage { + pub fn path(&self) -> &Path { + self.path.as_path() + } +} /// Mapped from `.git-next.toml` file in target repo /// Is also derived from the optional parameters in `git-next-server.toml` at @@ -181,7 +186,7 @@ impl Display for ForgeConfig { pub struct ServerRepoConfig { repo: String, branch: String, - gitdir: Option, + gitdir: Option, // TODO: (#71) use this when supplied main: Option, next: Option, dev: Option, @@ -229,6 +234,11 @@ impl Display for ForgeName { write!(f, "{}", self.0) } } +impl From<&ForgeName> for PathBuf { + fn from(value: &ForgeName) -> Self { + Self::from(&value.0) + } +} /// The hostname of a forge #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/src/server/gitforge/errors.rs b/src/server/gitforge/errors.rs index 9530e51f..de7064ff 100644 --- a/src/server/gitforge/errors.rs +++ b/src/server/gitforge/errors.rs @@ -47,6 +47,8 @@ pub enum RepoCloneError { Wait(std::io::Error), Spawn(std::io::Error), Validation(String), + GixClone(Box), + GixFetch(Box), } impl std::error::Error for RepoCloneError {} impl std::fmt::Display for RepoCloneError { @@ -57,6 +59,18 @@ impl std::fmt::Display for RepoCloneError { Self::Wait(err) => write!(f, "Waiting for command: {:?}", err), Self::Spawn(err) => write!(f, "Spawning comming: {:?}", err), Self::Validation(err) => write!(f, "Validation: {}", err), + Self::GixClone(err) => write!(f, "Clone: {:?}", err), + Self::GixFetch(err) => write!(f, "Fetch: {:?}", err), } } } +impl From for RepoCloneError { + fn from(value: gix::clone::Error) -> Self { + Self::GixClone(Box::new(value)) + } +} +impl From for RepoCloneError { + fn from(value: gix::clone::fetch::Error) -> Self { + Self::GixFetch(Box::new(value)) + } +} diff --git a/src/server/gitforge/forgejo/mod.rs b/src/server/gitforge/forgejo/mod.rs index 4638af0b..d078e1c3 100644 --- a/src/server/gitforge/forgejo/mod.rs +++ b/src/server/gitforge/forgejo/mod.rs @@ -5,7 +5,7 @@ mod repo; use actix::prelude::*; use kxio::network::{self, Network}; -use tracing::{error, warn}; +use tracing::{error, info, warn}; use crate::server::{ actors::repo::RepoActor, @@ -102,11 +102,14 @@ impl super::ForgeLike for ForgeJoEnv { fn repo_clone(&self, gitdir: GitDir) -> Result<(), RepoCloneError> { if gitdir.exists() { + info!(?gitdir, "Gitdir already exists - validating..."); gitdir .validate(&self.repo_details) .map_err(|e| RepoCloneError::Validation(e.to_string())) + .inspect(|_| info!(?gitdir, "Validation - OK")) } else { - repo::clone(&self.repo_details, gitdir) + info!(?gitdir, "Gitdir doesn't exists - cloning..."); + repo::clone(&self.repo_details, gitdir).inspect(|_| info!("Cloned - OK")) } } } diff --git a/src/server/gitforge/forgejo/repo/clone.rs b/src/server/gitforge/forgejo/repo/clone.rs index e6604d99..c4b99ec4 100644 --- a/src/server/gitforge/forgejo/repo/clone.rs +++ b/src/server/gitforge/forgejo/repo/clone.rs @@ -1,4 +1,6 @@ -use tracing::{info, warn}; +use std::{ops::Deref, sync::atomic::AtomicBool}; + +use tracing::info; use crate::server::{ config::{GitDir, RepoDetails}, @@ -6,35 +8,13 @@ use crate::server::{ }; pub fn clone(repo_details: &RepoDetails, gitdir: GitDir) -> Result<(), RepoCloneError> { - let origin = repo_details.origin(); - // INFO: never log the command as it contains the API token within the 'origin' use secrecy::ExposeSecret; - let command: secrecy::Secret = format!( - "/usr/bin/git clone --bare -- {} {}", - origin.expose_secret(), - gitdir - ) - .into(); - let repo_name = &repo_details.repo_alias; - info!( - %repo_name, - %gitdir, - "Cloning" - ); - match gix::command::prepare(command.expose_secret()) - .with_shell_allow_argument_splitting() - .spawn() - { - Ok(mut child) => match child.wait() { - Ok(_) => Ok(()), - Err(err) => { - warn!(?err, "Failed (wait)"); - Err(RepoCloneError::Wait(err)) - } - }, - Err(err) => { - warn!(?err, "Failed (spawn)"); - Err(RepoCloneError::Spawn(err)) - } - } + let origin = repo_details.origin(); + info!("Cloning"); + let (repository, _outcome) = + gix::prepare_clone_bare(origin.expose_secret().as_str(), gitdir.deref())? + .fetch_only(gix::progress::Discard, &AtomicBool::new(false))?; + info!(?repository, "Cloned"); + + Ok(()) // TODO: (#69) return Repository inside a newtype to store in the RepoActor for reuse else where } diff --git a/src/server/mod.rs b/src/server/mod.rs index 0f77f296..168b34b2 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -21,6 +21,16 @@ use crate::{ use self::{actors::repo::RepoActor, config::ServerRepoConfig}; +#[derive(Debug, derive_more::Display, derive_more::From)] +pub enum Error { + #[display("Failed to create data directories")] + FailedToCreateDataDirectory(kxio::fs::Error), + + #[display("The forge data path is not a directory: {path:?}")] + ForgeDirIsNotDirectory { path: PathBuf }, +} +type Result = core::result::Result; + pub fn init(fs: FileSystem) { let file_name = "git-next-server.toml"; let pathbuf = PathBuf::from(file_name); @@ -44,10 +54,7 @@ pub fn init(fs: FileSystem) { } pub async fn start(fs: FileSystem, net: Network) { - let Ok(_) = init_logging() else { - eprintln!("Failed to initialize logging."); - return; - }; + init_logging(); info!("Starting Server..."); let server_config = match config::ServerConfig::load(&fs) { Ok(server_config) => server_config, @@ -56,11 +63,24 @@ pub async fn start(fs: FileSystem, net: Network) { return; } }; + // create data dir if missing + let dir = server_config.storage().path(); + if !dir.exists() { + if let Err(err) = fs.dir_create(dir) { + error!(?err, ?dir, "Failed to create server storage directory"); + return; + } + } let webhook_router = webhook::WebhookRouter::new().start(); let webhook = server_config.webhook(); let server_storage = server_config.storage(); + if let Err(err) = create_forge_data_directories(&server_config, &fs, &dir) { + error!(?err, "Failure creating forge data directories"); + return; + } + server_config .forges() .flat_map(|(forge_name, forge_config)| { @@ -75,6 +95,26 @@ pub async fn start(fs: FileSystem, net: Network) { drop(webhook_server); } +fn create_forge_data_directories( + server_config: &config::ServerConfig, + fs: &FileSystem, + server_dir: &&std::path::Path, +) -> Result<()> { + for (forge_name, _forge_config) in server_config.forges() { + let forge_dir: PathBuf = (&forge_name).into(); + let path = server_dir.join(&forge_dir); + if fs.path_exists(&path)? { + if !fs.path_is_dir(&path)? { + return Err(Error::ForgeDirIsNotDirectory { path }); + } + } else { + fs.dir_create_all(&path)?; + } + } + + Ok(()) +} + fn create_forge_repos( forge_config: &ForgeConfig, forge_name: ForgeName, @@ -140,7 +180,7 @@ fn start_actor( (repo_alias, addr) } -pub fn init_logging() -> Result<(), tracing::subscriber::SetGlobalDefaultError> { +pub fn init_logging() { use tracing_subscriber::prelude::*; let subscriber = tracing_subscriber::fmt::layer() @@ -153,5 +193,4 @@ pub fn init_logging() -> Result<(), tracing::subscriber::SetGlobalDefaultError> .with(console_subscriber::ConsoleLayer::builder().spawn()) .with(subscriber) .init(); - Ok(()) }