| //! Utilities for handling git repositories, mainly around |
| //! authentication/cloning. |
| |
| use crate::context::ProgressWhen; |
| use crate::sources::git::fetch::RemoteKind; |
| use crate::sources::git::oxide; |
| use crate::sources::git::oxide::cargo_config_to_gitoxide_overrides; |
| use crate::sources::git::source::GitSource; |
| use crate::sources::source::Source as _; |
| use crate::util::HumanBytes; |
| use crate::util::errors::{CargoResult, GitCliError}; |
| use crate::util::network::http::http_handle; |
| use crate::util::network::http::needs_custom_http_transport; |
| use crate::util::{GlobalContext, IntoUrl, MetricsCounter, Progress, network}; |
| use crate::workspace::{GitReference, SourceId}; |
| |
| use anyhow::{Context as _, anyhow}; |
| use cargo_util::{ProcessBuilder, paths}; |
| use cargo_util_terminal::Verbosity; |
| use git2::{ErrorClass, ObjectType, Oid}; |
| use http::{Request, StatusCode}; |
| use tracing::{debug, info}; |
| use url::Url; |
| |
| use std::borrow::Cow; |
| use std::path::{Path, PathBuf}; |
| use std::process::Command; |
| use std::str; |
| use std::sync::Once; |
| use std::sync::atomic::{AtomicBool, Ordering}; |
| use std::time::{Duration, Instant}; |
| |
| /// A file indicates that if present, `git reset` has been done and a repo |
| /// checkout is ready to go. See [`GitCheckout::reset`] for why we need this. |
| const CHECKOUT_READY_LOCK: &str = ".cargo-ok"; |
| |
| /// A short abbreviated OID. |
| /// |
| /// Exists for avoiding extra allocations in [`GitDatabase::to_short_id`]. |
| pub struct GitShortID(git2::Buf); |
| |
| impl GitShortID { |
| /// Views the short ID as a `str`. |
| pub fn as_str(&self) -> &str { |
| self.0.as_str().unwrap() |
| } |
| } |
| |
| /// A remote repository. It gets cloned into a local [`GitDatabase`]. |
| #[derive(PartialEq, Clone, Debug)] |
| pub struct GitRemote { |
| /// URL to a remote repository. |
| /// |
| /// This may differ from the [`SourceId`] URL when the original URL |
| /// can't be represented as a WHATWG [`Url`], for example SCP-like URLs. |
| /// See <https://github.com/rust-lang/cargo/issues/16740>. |
| url: String, |
| } |
| |
| /// A local clone of a remote repository's database. Multiple [`GitCheckout`]s |
| /// can be cloned from a single [`GitDatabase`]. |
| pub struct GitDatabase { |
| /// The remote repository where this database is fetched from. |
| remote: GitRemote, |
| /// Path to the root of the underlying Git repository on the local filesystem. |
| path: PathBuf, |
| /// Underlying Git repository instance for this database. |
| repo: git2::Repository, |
| } |
| |
| /// A local checkout of a particular revision from a [`GitDatabase`]. |
| pub struct GitCheckout<'a> { |
| /// The git database where this checkout is cloned from. |
| database: &'a GitDatabase, |
| /// Path to the root of the underlying Git repository on the local filesystem. |
| path: PathBuf, |
| /// The git revision this checkout is for. |
| revision: git2::Oid, |
| /// Underlying Git repository instance for this checkout. |
| repo: git2::Repository, |
| } |
| |
| impl GitRemote { |
| /// Creates an instance for a remote repository URL. |
| pub fn new(url: &Url) -> GitRemote { |
| GitRemote { |
| url: url.as_str().to_owned(), |
| } |
| } |
| |
| /// Creates an instance with an URL that may not be a valid WHATWG URL. |
| /// |
| /// This is needed because [`SourceId`] hasn't yet supported SCP-like URLs. |
| pub(super) fn new_from_str(url: String) -> GitRemote { |
| GitRemote { url } |
| } |
| |
| /// Gets the remote repository URL. |
| pub fn url(&self) -> &str { |
| &self.url |
| } |
| |
| /// Fetches and checkouts to a reference or a revision from this remote |
| /// into a local path. |
| /// |
| /// This ensures that it gets the up-to-date commit when a named reference |
| /// is given (tag, branch, refs/*). Thus, network connection is involved. |
| /// |
| /// If we have a previous instance of [`GitDatabase`] then fetch into that |
| /// if we can. If that can successfully load our revision then we've |
| /// populated the database with the latest version of `reference`, so |
| /// return that database and the rev we resolve to. |
| pub fn checkout( |
| &self, |
| into: &Path, |
| db: Option<GitDatabase>, |
| manifest_reference: &GitReference, |
| reference: &GitReference, |
| gctx: &GlobalContext, |
| ) -> CargoResult<(GitDatabase, git2::Oid)> { |
| if let Some(mut db) = db { |
| fetch( |
| &mut db.repo, |
| self.url(), |
| manifest_reference, |
| reference, |
| gctx, |
| RemoteKind::GitDependency, |
| ) |
| .with_context(|| format!("failed to fetch into: {}", into.display()))?; |
| |
| if let Some(rev) = resolve_ref(reference, &db.repo).ok() { |
| return Ok((db, rev)); |
| } |
| } |
| |
| // Otherwise start from scratch to handle corrupt git repositories. |
| // After our fetch (which is interpreted as a clone now) we do the same |
| // resolution to figure out what we cloned. |
| if into.exists() { |
| paths::remove_dir_all(into)?; |
| } |
| paths::create_dir_all(into)?; |
| let mut repo = init(into, true)?; |
| fetch( |
| &mut repo, |
| self.url(), |
| manifest_reference, |
| reference, |
| gctx, |
| RemoteKind::GitDependency, |
| ) |
| .with_context(|| format!("failed to clone into: {}", into.display()))?; |
| let rev = resolve_ref(reference, &repo)?; |
| |
| Ok(( |
| GitDatabase { |
| remote: self.clone(), |
| path: into.to_path_buf(), |
| repo, |
| }, |
| rev, |
| )) |
| } |
| |
| /// Creates a [`GitDatabase`] of this remote at `db_path`. |
| pub fn db_at(&self, db_path: &Path) -> CargoResult<GitDatabase> { |
| let repo = git2::Repository::open(db_path)?; |
| Ok(GitDatabase { |
| remote: self.clone(), |
| path: db_path.to_path_buf(), |
| repo, |
| }) |
| } |
| } |
| |
| impl GitDatabase { |
| /// Checkouts to a revision at `dest`ination from this database. |
| #[tracing::instrument(skip(self, gctx))] |
| pub fn copy_to( |
| &self, |
| rev: git2::Oid, |
| dest: &Path, |
| gctx: &GlobalContext, |
| quiet: bool, |
| ) -> CargoResult<GitCheckout<'_>> { |
| // If the existing checkout exists, and it is fresh, use it. |
| // A non-fresh checkout can happen if the checkout operation was |
| // interrupted. In that case, the checkout gets deleted and a new |
| // clone is created. |
| let checkout = match git2::Repository::open(dest) |
| .ok() |
| .map(|repo| GitCheckout::new(self, rev, repo)) |
| .filter(|co| co.is_fresh()) |
| { |
| Some(co) => co, |
| None => { |
| let (checkout, guard) = GitCheckout::clone_into(dest, self, rev, gctx)?; |
| checkout.update_submodules(gctx, quiet)?; |
| guard.mark_ok()?; |
| checkout |
| } |
| }; |
| |
| Ok(checkout) |
| } |
| |
| /// Get a short OID for a `revision`, usually 7 chars or more if ambiguous. |
| pub fn to_short_id(&self, revision: git2::Oid) -> CargoResult<GitShortID> { |
| let obj = self.repo.find_object(revision, None)?; |
| Ok(GitShortID(obj.short_id()?)) |
| } |
| |
| /// Checks if the database contains the object of this `oid`.. |
| pub fn contains(&self, oid: git2::Oid) -> bool { |
| self.repo.revparse_single(&oid.to_string()).is_ok() |
| } |
| |
| /// [`resolve_ref`]s this reference with this database. |
| pub fn resolve(&self, r: &GitReference) -> CargoResult<git2::Oid> { |
| resolve_ref(r, &self.repo) |
| } |
| } |
| |
| /// Resolves [`GitReference`] to an object ID with objects the `repo` currently has. |
| pub fn resolve_ref(gitref: &GitReference, repo: &git2::Repository) -> CargoResult<git2::Oid> { |
| let id = match gitref { |
| // Note that we resolve the named tag here in sync with where it's |
| // fetched into via `fetch` below. |
| GitReference::Tag(s) => (|| -> CargoResult<git2::Oid> { |
| let refname = format!("refs/remotes/origin/tags/{}", s); |
| let id = repo.refname_to_id(&refname)?; |
| let obj = repo.find_object(id, None)?; |
| let obj = obj.peel(ObjectType::Commit)?; |
| Ok(obj.id()) |
| })() |
| .with_context(|| format!("failed to find tag `{}`", s))?, |
| |
| // Resolve the remote name since that's all we're configuring in |
| // `fetch` below. |
| GitReference::Branch(s) => { |
| let name = format!("origin/{}", s); |
| let b = repo |
| .find_branch(&name, git2::BranchType::Remote) |
| .with_context(|| format!("failed to find branch `{}`", s))?; |
| b.get() |
| .target() |
| .ok_or_else(|| anyhow::format_err!("branch `{}` did not have a target", s))? |
| } |
| |
| // We'll be using the HEAD commit |
| GitReference::DefaultBranch => { |
| let head_id = repo.refname_to_id("refs/remotes/origin/HEAD")?; |
| let head = repo.find_object(head_id, None)?; |
| head.peel(ObjectType::Commit)?.id() |
| } |
| |
| GitReference::Rev(s) => { |
| let obj = repo.revparse_single(s)?; |
| match obj.as_tag() { |
| Some(tag) => tag.target_id(), |
| None => obj.id(), |
| } |
| } |
| }; |
| Ok(id) |
| } |
| |
| impl<'a> GitCheckout<'a> { |
| /// Creates an instance of [`GitCheckout`]. This doesn't imply the checkout |
| /// is done. Use [`GitCheckout::is_fresh`] to check. |
| /// |
| /// * The `database` is where this checkout is from. |
| /// * The `repo` will be the checked out Git repository. |
| fn new( |
| database: &'a GitDatabase, |
| revision: git2::Oid, |
| repo: git2::Repository, |
| ) -> GitCheckout<'a> { |
| let path = repo.workdir().unwrap_or_else(|| repo.path()); |
| GitCheckout { |
| path: path.to_path_buf(), |
| database, |
| revision, |
| repo, |
| } |
| } |
| |
| /// Gets the remote repository URL. |
| fn remote_url(&self) -> &str { |
| self.database.remote.url() |
| } |
| |
| /// Clone a repo for a `revision` into a local path from a `database`. |
| /// This is a filesystem-to-filesystem clone. |
| fn clone_into( |
| into: &Path, |
| database: &'a GitDatabase, |
| revision: git2::Oid, |
| gctx: &GlobalContext, |
| ) -> CargoResult<(GitCheckout<'a>, CheckoutGuard)> { |
| let dirname = into.parent().unwrap(); |
| paths::create_dir_all(&dirname)?; |
| if into.exists() { |
| paths::remove_dir_all(into)?; |
| } |
| |
| // we're doing a local filesystem-to-filesystem clone so there should |
| // be no need to respect global configuration options, so pass in |
| // an empty instance of `git2::Config` below. |
| let git_config = git2::Config::new()?; |
| |
| // Clone the repository, but make sure we use the "local" option in |
| // libgit2 which will attempt to use hardlinks to set up the database. |
| // This should speed up the clone operation quite a bit if it works. |
| // |
| // Note that we still use the same fetch options because while we don't |
| // need authentication information we may want progress bars and such. |
| let url = database.path.into_url()?; |
| let mut repo = None; |
| with_fetch_options(&git_config, url.as_str(), gctx, &mut |fopts| { |
| let mut checkout = git2::build::CheckoutBuilder::new(); |
| checkout.dry_run(); // we'll do this below during a `reset` |
| |
| let r = git2::build::RepoBuilder::new() |
| // use hard links and/or copy the database, we're doing a |
| // filesystem clone so this'll speed things up quite a bit. |
| .clone_local(git2::build::CloneLocal::Local) |
| .with_checkout(checkout) |
| .fetch_options(fopts) |
| .clone(url.as_str(), into)?; |
| // `git2` doesn't seem to handle shallow repos correctly when doing |
| // a local clone. Fortunately all that's needed is the copy of the |
| // one file that defines the shallow boundary, the commits which |
| // have their parents omitted as part of the shallow clone. |
| // |
| // TODO(git2): remove this when git2 supports shallow clone correctly |
| if database.repo.is_shallow() { |
| std::fs::copy( |
| database.repo.path().join("shallow"), |
| r.path().join("shallow"), |
| )?; |
| } |
| repo = Some(r); |
| Ok(()) |
| })?; |
| let repo = repo.unwrap(); |
| |
| let checkout = GitCheckout::new(database, revision, repo); |
| let guard = checkout.reset(gctx)?; |
| Ok((checkout, guard)) |
| } |
| |
| /// Checks if the `HEAD` of this checkout points to the expected revision. |
| fn is_fresh(&self) -> bool { |
| match self.repo.revparse_single("HEAD") { |
| Ok(ref head) if head.id() == self.revision => { |
| // See comments in reset() for why we check this |
| self.path.join(CHECKOUT_READY_LOCK).exists() |
| } |
| _ => false, |
| } |
| } |
| |
| /// Similar to [`reset()`]. This roughly performs `git reset --hard` to the |
| /// revision of this checkout, with additional interrupt protection by a |
| /// dummy file [`CHECKOUT_READY_LOCK`]. |
| /// |
| /// If we're interrupted while performing a `git reset` (e.g., we die |
| /// because of a signal) Cargo needs to be sure to try to check out this |
| /// repo again on the next go-round. |
| /// |
| /// To enable this we have a dummy file in our checkout, [`.cargo-ok`], |
| /// which if present means that the repo has been successfully reset and is |
| /// ready to go. Hence if we start to do a reset, we make sure this file |
| /// *doesn't* exist. The caller of [`reset`] has an option to perform additional operations |
| /// (e.g. submodule update) before marking the check-out as ready. |
| /// |
| /// [`.cargo-ok`]: CHECKOUT_READY_LOCK |
| fn reset(&self, gctx: &GlobalContext) -> CargoResult<CheckoutGuard> { |
| let guard = CheckoutGuard::guard(&self.path); |
| info!("reset {} to {}", self.repo.path().display(), self.revision); |
| |
| // Ensure libgit2 won't mess with newlines when we vendor. |
| if let Ok(mut git_config) = self.repo.config() { |
| git_config.set_bool("core.autocrlf", false)?; |
| } |
| |
| let object = self.repo.find_object(self.revision, None)?; |
| reset(&self.repo, &object, gctx)?; |
| |
| Ok(guard) |
| } |
| |
| /// Like `git submodule update --recursive` but for this git checkout. |
| /// |
| /// This function respects `submodule.<name>.update = none`[^1] git config. |
| /// Submodules set to `none` won't be fetched. |
| /// |
| /// [^1]: <https://git-scm.com/docs/git-submodule#Documentation/git-submodule.txt-none> |
| fn update_submodules(&self, gctx: &GlobalContext, quiet: bool) -> CargoResult<()> { |
| return update_submodules(&self.repo, gctx, quiet, self.remote_url()); |
| |
| /// Recursive helper for [`GitCheckout::update_submodules`]. |
| fn update_submodules( |
| repo: &git2::Repository, |
| gctx: &GlobalContext, |
| quiet: bool, |
| parent_remote_url: &str, |
| ) -> CargoResult<()> { |
| debug!("update submodules for: {:?}", repo.workdir().unwrap()); |
| |
| for mut child in repo.submodules()? { |
| update_submodule(repo, &mut child, gctx, quiet, parent_remote_url).with_context( |
| || { |
| format!( |
| "failed to update submodule `{}`", |
| child.name().unwrap_or("") |
| ) |
| }, |
| )?; |
| } |
| Ok(()) |
| } |
| |
| /// Update a single Git submodule, and recurse into its submodules. |
| fn update_submodule( |
| parent: &git2::Repository, |
| child: &mut git2::Submodule<'_>, |
| gctx: &GlobalContext, |
| quiet: bool, |
| parent_remote_url: &str, |
| ) -> CargoResult<()> { |
| child.init(false)?; |
| |
| let child_url_str = child |
| .url() |
| .with_context(|| { |
| format!("failed to update submodule `{}`", child.path().display()) |
| })? |
| .ok_or_else(|| { |
| anyhow::format_err!( |
| "unable to update submodule `{}` without a path", |
| child.name().unwrap_or("") |
| ) |
| })?; |
| |
| // Skip the submodule if the config says not to update it. |
| if child.update_strategy() == git2::SubmoduleUpdate::None { |
| gctx.shell().status( |
| "Skipping", |
| format!( |
| "git submodule `{}` due to update strategy in .gitmodules", |
| child_url_str |
| ), |
| )?; |
| return Ok(()); |
| } |
| |
| let child_remote_url = absolute_submodule_url(parent_remote_url, child_url_str)?; |
| |
| // A submodule which is listed in .gitmodules but not actually |
| // checked out will not have a head id, so we should ignore it. |
| let Some(head) = child.head_id() else { |
| return Ok(()); |
| }; |
| |
| // If the submodule hasn't been checked out yet, we need to |
| // clone it. If it has been checked out and the head is the same |
| // as the submodule's head, then we can skip an update and keep |
| // recursing. |
| let head_and_repo = child.open().and_then(|repo| { |
| let target = repo.head()?.target(); |
| Ok((target, repo)) |
| }); |
| let repo = match head_and_repo { |
| Ok((head, repo)) => { |
| if child.head_id() == head { |
| return update_submodules(&repo, gctx, quiet, &child_remote_url); |
| } |
| repo |
| } |
| Err(..) => { |
| let path = parent.workdir().unwrap().join(child.path()); |
| let _ = paths::remove_dir_all(&path); |
| init(&path, false)? |
| } |
| }; |
| // Fetch submodule database and checkout to target revision |
| let reference = GitReference::Rev(head.to_string()); |
| |
| // SCP-like URL is not a WHATWG Standard URL. |
| // `url` crate can't parse SCP-like URLs. |
| // We convert to `ssh://` for SourceId, |
| // but preserve the original URL for fetch to maintain correct semantics |
| // See <https://github.com/rust-lang/cargo/issues/16740> |
| let (source_url, fetch_url) = match child_remote_url.as_ref().into_url() { |
| Ok(url) => (url, None), |
| Err(_) => { |
| let ssh_url = scp_to_ssh(&child_remote_url) |
| .ok_or_else(|| anyhow::format_err!("invalid url `{child_remote_url}`"))? |
| .as_str() |
| .into_url()?; |
| (ssh_url, Some(child_remote_url.into_owned())) |
| } |
| }; |
| |
| // GitSource created from SourceId without git precise will result to |
| // locked_rev being Deferred and fetch_db always try to fetch if online |
| let source_id = |
| SourceId::for_git(&source_url, reference)?.with_git_precise(Some(head.to_string())); |
| |
| let mut source = match &fetch_url { |
| Some(url) => GitSource::new_for_submodule(source_id, url.to_owned(), gctx)?, |
| None => GitSource::new(source_id, gctx)?, |
| }; |
| source.set_quiet(quiet); |
| |
| let (db, actual_rev) = source.fetch_db(true).with_context(|| { |
| let name = child.name().unwrap_or(""); |
| let url = fetch_url.unwrap_or_else(|| source_url.to_string()); |
| format!("failed to fetch submodule `{name}` from {url}") |
| })?; |
| db.copy_to(actual_rev, repo.path(), gctx, quiet)?; |
| Ok(()) |
| } |
| } |
| } |
| |
| /// See [`GitCheckout::reset`] for rationale on this type. |
| #[must_use] |
| struct CheckoutGuard { |
| ok_file: PathBuf, |
| } |
| |
| impl CheckoutGuard { |
| fn guard(path: &Path) -> Self { |
| let ok_file = path.join(CHECKOUT_READY_LOCK); |
| let _ = paths::remove_file(&ok_file); |
| Self { ok_file } |
| } |
| |
| fn mark_ok(self) -> CargoResult<()> { |
| let _ = paths::create(self.ok_file)?; |
| Ok(()) |
| } |
| } |
| |
| /// Constructs an absolute URL for a child submodule URL with its parent base URL. |
| /// |
| /// Git only assumes a submodule URL is a relative path if it starts with `./` |
| /// or `../` [^1]. To fetch the correct repo, we need to construct an absolute |
| /// submodule URL. |
| /// |
| /// At this moment it comes with some limitations: |
| /// |
| /// * GitHub doesn't accept non-normalized URLs with relative paths. |
| /// (`ssh://git@github.com/rust-lang/cargo.git/relative/..` is invalid) |
| /// * `url` crate cannot parse SCP-like URLs. |
| /// (`git@github.com:rust-lang/cargo.git` is not a valid WHATWG URL) |
| /// |
| /// To overcome these, this patch always tries [`Url::parse`] first to normalize |
| /// the path. If it couldn't, append the relative path and/or convert SCP-like URLs |
| /// to ssh:// format as the last resorts and pray the remote git service supports |
| /// non-normalized URLs. |
| /// |
| /// See also rust-lang/cargo#12404 and rust-lang/cargo#12295. |
| /// |
| /// [^1]: <https://git-scm.com/docs/git-submodule> |
| fn absolute_submodule_url<'s>(base_url: &str, submodule_url: &'s str) -> CargoResult<Cow<'s, str>> { |
| let absolute_url = if ["./", "../"].iter().any(|p| submodule_url.starts_with(p)) { |
| match Url::parse(base_url) { |
| Ok(mut base_url) => { |
| let path = base_url.path(); |
| if !path.ends_with('/') { |
| base_url.set_path(&format!("{path}/")); |
| } |
| let absolute_url = base_url.join(submodule_url).with_context(|| { |
| format!( |
| "failed to parse relative child submodule url `{submodule_url}` \ |
| using parent base url `{base_url}`" |
| ) |
| })?; |
| Cow::from(absolute_url.to_string()) |
| } |
| Err(_) => { |
| let mut absolute_url = base_url.to_string(); |
| if !absolute_url.ends_with('/') { |
| absolute_url.push('/'); |
| } |
| absolute_url.push_str(submodule_url); |
| Cow::from(absolute_url) |
| } |
| } |
| } else { |
| Cow::from(submodule_url) |
| }; |
| |
| Ok(absolute_url) |
| } |
| |
| /// Converts an SCP-like URL to `ssh://` format. |
| fn scp_to_ssh(url: &str) -> Option<String> { |
| let mut gix_url = gix::url::parse(gix::bstr::BStr::new(url.as_bytes())).ok()?; |
| if gix_url.serialize_alternative_form && gix_url.scheme == gix::url::Scheme::Ssh { |
| gix_url.serialize_alternative_form = false; |
| Some(gix_url.to_bstring().to_string()) |
| } else { |
| None |
| } |
| } |
| |
| /// Prepare the authentication callbacks for cloning a git repository. |
| /// |
| /// The main purpose of this function is to construct the "authentication |
| /// callback" which is used to clone a repository. This callback will attempt to |
| /// find the right authentication on the system (without user input) and will |
| /// guide libgit2 in doing so. |
| /// |
| /// The callback is provided `allowed` types of credentials, and we try to do as |
| /// much as possible based on that: |
| /// |
| /// * Prioritize SSH keys from the local ssh agent as they're likely the most |
| /// reliable. The username here is prioritized from the credential |
| /// callback, then from whatever is configured in git itself, and finally |
| /// we fall back to the generic user of `git`. |
| /// |
| /// * If a username/password is allowed, then we fallback to git2-rs's |
| /// implementation of the credential helper. This is what is configured |
| /// with `credential.helper` in git, and is the interface for the macOS |
| /// keychain, for example. |
| /// |
| /// * After the above two have failed, we just kinda grapple attempting to |
| /// return *something*. |
| /// |
| /// If any form of authentication fails, libgit2 will repeatedly ask us for |
| /// credentials until we give it a reason to not do so. To ensure we don't |
| /// just sit here looping forever we keep track of authentications we've |
| /// attempted and we don't try the same ones again. |
| fn with_authentication<T, F>( |
| gctx: &GlobalContext, |
| url: &str, |
| cfg: &git2::Config, |
| mut f: F, |
| ) -> CargoResult<T> |
| where |
| F: FnMut(&mut git2::Credentials<'_>) -> CargoResult<T>, |
| { |
| let mut cred_helper = git2::CredentialHelper::new(url); |
| cred_helper.config(cfg); |
| |
| let mut ssh_username_requested = false; |
| let mut cred_helper_bad = None; |
| let mut ssh_agent_attempts = Vec::new(); |
| let mut any_attempts = false; |
| let mut tried_sshkey = false; |
| let mut url_attempt = None; |
| |
| let orig_url = url; |
| let mut res = f(&mut |url, username, allowed| { |
| any_attempts = true; |
| if url != orig_url { |
| url_attempt = Some(url.to_string()); |
| } |
| // libgit2's "USERNAME" authentication actually means that it's just |
| // asking us for a username to keep going. This is currently only really |
| // used for SSH authentication and isn't really an authentication type. |
| // The logic currently looks like: |
| // |
| // let user = ...; |
| // if (user.is_null()) |
| // user = callback(USERNAME, null, ...); |
| // |
| // callback(SSH_KEY, user, ...) |
| // |
| // So if we're being called here then we know that (a) we're using ssh |
| // authentication and (b) no username was specified in the URL that |
| // we're trying to clone. We need to guess an appropriate username here, |
| // but that may involve a few attempts. Unfortunately we can't switch |
| // usernames during one authentication session with libgit2, so to |
| // handle this we bail out of this authentication session after setting |
| // the flag `ssh_username_requested`, and then we handle this below. |
| if allowed.contains(git2::CredentialType::USERNAME) { |
| debug_assert!(username.is_none()); |
| ssh_username_requested = true; |
| return Err(git2::Error::from_str("gonna try usernames later")); |
| } |
| |
| // An "SSH_KEY" authentication indicates that we need some sort of SSH |
| // authentication. This can currently either come from the ssh-agent |
| // process or from a raw in-memory SSH key. Cargo only supports using |
| // ssh-agent currently. |
| // |
| // If we get called with this then the only way that should be possible |
| // is if a username is specified in the URL itself (e.g., `username` is |
| // Some), hence the unwrap() here. We try custom usernames down below. |
| if allowed.contains(git2::CredentialType::SSH_KEY) && !tried_sshkey { |
| // If ssh-agent authentication fails, libgit2 will keep |
| // calling this callback asking for other authentication |
| // methods to try. Make sure we only try ssh-agent once, |
| // to avoid looping forever. |
| tried_sshkey = true; |
| let username = username.unwrap(); |
| debug_assert!(!ssh_username_requested); |
| ssh_agent_attempts.push(username.to_string()); |
| return git2::Cred::ssh_key_from_agent(username); |
| } |
| |
| // Sometimes libgit2 will ask for a username/password in plaintext. This |
| // is where Cargo would have an interactive prompt if we supported it, |
| // but we currently don't! Right now the only way we support fetching a |
| // plaintext password is through the `credential.helper` support, so |
| // fetch that here. |
| // |
| // If ssh-agent authentication fails, libgit2 will keep calling this |
| // callback asking for other authentication methods to try. Check |
| // cred_helper_bad to make sure we only try the git credential helper |
| // once, to avoid looping forever. |
| if allowed.contains(git2::CredentialType::USER_PASS_PLAINTEXT) && cred_helper_bad.is_none() |
| { |
| let r = git2::Cred::credential_helper(cfg, url, username); |
| cred_helper_bad = Some(r.is_err()); |
| return r; |
| } |
| |
| // I'm... not sure what the DEFAULT kind of authentication is, but seems |
| // easy to support? |
| if allowed.contains(git2::CredentialType::DEFAULT) { |
| return git2::Cred::default(); |
| } |
| |
| // Whelp, we tried our best |
| Err(git2::Error::from_str("no authentication methods succeeded")) |
| }); |
| |
| // Ok, so if it looks like we're going to be doing ssh authentication, we |
| // want to try a few different usernames as one wasn't specified in the URL |
| // for us to use. In order, we'll try: |
| // |
| // * A credential helper's username for this URL, if available. |
| // * This account's username. |
| // * "git" |
| // |
| // We have to restart the authentication session each time (due to |
| // constraints in libssh2 I guess? maybe this is inherent to ssh?), so we |
| // call our callback, `f`, in a loop here. |
| if ssh_username_requested { |
| debug_assert!(res.is_err()); |
| let mut attempts = vec![String::from("git")]; |
| if let Ok(s) = gctx.get_env("USER").or_else(|_| gctx.get_env("USERNAME")) { |
| attempts.push(s.to_string()); |
| } |
| if let Some(ref s) = cred_helper.username { |
| attempts.push(s.clone()); |
| } |
| |
| while let Some(s) = attempts.pop() { |
| // We should get `USERNAME` first, where we just return our attempt, |
| // and then after that we should get `SSH_KEY`. If the first attempt |
| // fails we'll get called again, but we don't have another option so |
| // we bail out. |
| let mut attempts = 0; |
| res = f(&mut |_url, username, allowed| { |
| if allowed.contains(git2::CredentialType::USERNAME) { |
| return git2::Cred::username(&s); |
| } |
| if allowed.contains(git2::CredentialType::SSH_KEY) { |
| debug_assert_eq!(Some(&s[..]), username); |
| attempts += 1; |
| if attempts == 1 { |
| ssh_agent_attempts.push(s.to_string()); |
| return git2::Cred::ssh_key_from_agent(&s); |
| } |
| } |
| Err(git2::Error::from_str("no authentication methods succeeded")) |
| }); |
| |
| // If we made two attempts then that means: |
| // |
| // 1. A username was requested, we returned `s`. |
| // 2. An ssh key was requested, we returned to look up `s` in the |
| // ssh agent. |
| // 3. For whatever reason that lookup failed, so we were asked again |
| // for another mode of authentication. |
| // |
| // Essentially, if `attempts == 2` then in theory the only error was |
| // that this username failed to authenticate (e.g., no other network |
| // errors happened). Otherwise something else is funny so we bail |
| // out. |
| if attempts != 2 { |
| break; |
| } |
| } |
| } |
| let mut err = match res { |
| Ok(e) => return Ok(e), |
| Err(e) => e, |
| }; |
| |
| // In the case of an authentication failure (where we tried something) then |
| // we try to give a more helpful error message about precisely what we |
| // tried. |
| if any_attempts { |
| let mut msg = "failed to authenticate when downloading \ |
| repository" |
| .to_string(); |
| |
| if let Some(attempt) = &url_attempt { |
| if url != attempt { |
| msg.push_str(": "); |
| msg.push_str(attempt); |
| } |
| } |
| msg.push('\n'); |
| if !ssh_agent_attempts.is_empty() { |
| let names = ssh_agent_attempts |
| .iter() |
| .map(|s| format!("`{}`", s)) |
| .collect::<Vec<_>>() |
| .join(", "); |
| msg.push_str(&format!( |
| "\n* attempted ssh-agent authentication, but \ |
| no usernames succeeded: {}", |
| names |
| )); |
| } |
| if let Some(failed_cred_helper) = cred_helper_bad { |
| if failed_cred_helper { |
| msg.push_str( |
| "\n* attempted to find username/password via \ |
| git's `credential.helper` support, but failed", |
| ); |
| } else { |
| msg.push_str( |
| "\n* attempted to find username/password via \ |
| `credential.helper`, but maybe the found \ |
| credentials were incorrect", |
| ); |
| } |
| } |
| msg.push_str("\n\n"); |
| msg.push_str("if the git CLI succeeds then `net.git-fetch-with-cli` may help here\n"); |
| msg.push_str("https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli"); |
| err = err.context(msg); |
| |
| // Otherwise if we didn't even get to the authentication phase them we may |
| // have failed to set up a connection, in these cases hint on the |
| // `net.git-fetch-with-cli` configuration option. |
| } else if let Some(e) = err.downcast_ref::<git2::Error>() { |
| match e.class() { |
| ErrorClass::Net |
| | ErrorClass::Ssl |
| | ErrorClass::Submodule |
| | ErrorClass::FetchHead |
| | ErrorClass::Ssh |
| | ErrorClass::Http => { |
| let msg = format!( |
| concat!( |
| "network failure seems to have happened\n", |
| "if a proxy or similar is necessary `net.git-fetch-with-cli` may help here\n", |
| "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli", |
| "{}" |
| ), |
| note_github_pull_request(url).unwrap_or_default() |
| ); |
| err = err.context(msg); |
| } |
| ErrorClass::Callback => { |
| // This unwraps the git2 error. We're using the callback error |
| // specifically to convey errors from Rust land through the C |
| // callback interface. We don't need the `; class=Callback |
| // (26)` that gets tacked on to the git2 error message. |
| err = anyhow::format_err!("{}", e.message()); |
| } |
| _ => {} |
| } |
| } |
| |
| Err(err) |
| } |
| |
| /// `git reset --hard` to the given `obj` for the `repo`. |
| /// |
| /// The `obj` is a commit-ish to which the head should be moved. |
| #[tracing::instrument(skip_all)] |
| fn reset(repo: &git2::Repository, obj: &git2::Object<'_>, gctx: &GlobalContext) -> CargoResult<()> { |
| let mut pb = Progress::new("Checkout", gctx); |
| let mut opts = git2::build::CheckoutBuilder::new(); |
| opts.progress(|_, cur, max| { |
| drop(pb.tick(cur, max, "")); |
| }); |
| debug!("doing reset"); |
| repo.reset(obj, git2::ResetType::Hard, Some(&mut opts))?; |
| debug!("reset done"); |
| Ok(()) |
| } |
| |
| /// Prepares the callbacks for fetching a git repository. |
| /// |
| /// The main purpose of this function is to construct everything before a fetch. |
| /// This will attempt to setup a progress bar, the authentication for git, |
| /// ssh known hosts check, and the network retry mechanism. |
| /// |
| /// The callback is provided a fetch options, which can be used by the actual |
| /// git fetch. |
| pub fn with_fetch_options( |
| git_config: &git2::Config, |
| url: &str, |
| gctx: &GlobalContext, |
| cb: &mut dyn FnMut(git2::FetchOptions<'_>) -> CargoResult<()>, |
| ) -> CargoResult<()> { |
| let mut progress = Progress::new("Fetch", gctx); |
| let ssh_config = gctx.net_config()?.ssh.as_ref(); |
| let config_known_hosts = ssh_config.and_then(|ssh| ssh.known_hosts.as_ref()); |
| let diagnostic_home_config = gctx.diagnostic_home_config(); |
| network::retry::with_retry(gctx, || { |
| // Hack: libgit2 disallows overriding the error from check_cb since v1.8.0, |
| // so we store the error additionally and unwrap it later |
| let mut check_cb_result = Ok(()); |
| let auth_result = with_authentication(gctx, url, git_config, |f| { |
| let port = Url::parse(url).ok().and_then(|url| url.port()); |
| let mut last_update = Instant::now(); |
| let mut rcb = git2::RemoteCallbacks::new(); |
| // We choose `N=10` here to make a `300ms * 10slots ~= 3000ms` |
| // sliding window for tracking the data transfer rate (in bytes/s). |
| let mut counter = MetricsCounter::<10>::new(0, last_update); |
| rcb.credentials(f); |
| rcb.certificate_check(|cert, host| { |
| match super::known_hosts::certificate_check( |
| gctx, |
| cert, |
| host, |
| port, |
| config_known_hosts, |
| &diagnostic_home_config, |
| ) { |
| Ok(status) => Ok(status), |
| Err(e) => { |
| check_cb_result = Err(e); |
| // This is not really used because it'll be overridden by libgit2 |
| // See https://github.com/libgit2/libgit2/commit/9a9f220119d9647a352867b24b0556195cb26548 |
| Err(git2::Error::from_str( |
| "invalid or unknown remote ssh hostkey", |
| )) |
| } |
| } |
| }); |
| rcb.transfer_progress(|stats| { |
| let indexed_deltas = stats.indexed_deltas(); |
| let msg = if indexed_deltas > 0 { |
| // Resolving deltas. |
| format!( |
| ", ({}/{}) resolving deltas", |
| indexed_deltas, |
| stats.total_deltas() |
| ) |
| } else { |
| // Receiving objects. |
| // |
| // # Caveat |
| // |
| // Progress bar relies on git2 calling `transfer_progress` |
| // to update its transfer rate, but we cannot guarantee a |
| // periodic call of that callback. Thus if we don't receive |
| // any data for, say, 10 seconds, the rate will get stuck |
| // and never go down to 0B/s. |
| // In the future, we need to find away to update the rate |
| // even when the callback is not called. |
| let now = Instant::now(); |
| // Scrape a `received_bytes` to the counter every 300ms. |
| if now - last_update > Duration::from_millis(300) { |
| counter.add(stats.received_bytes(), now); |
| last_update = now; |
| } |
| let rate = HumanBytes(counter.rate() as u64); |
| format!(", {rate:.2}/s") |
| }; |
| progress |
| .tick(stats.indexed_objects(), stats.total_objects(), &msg) |
| .is_ok() |
| }); |
| |
| // Create a local anonymous remote in the repository to fetch the |
| // url |
| let mut opts = git2::FetchOptions::new(); |
| opts.remote_callbacks(rcb); |
| cb(opts) |
| }); |
| if auth_result.is_err() { |
| check_cb_result?; |
| } |
| auth_result?; |
| Ok(()) |
| }) |
| } |
| |
| /// Attempts to fetch the given git `reference` for a Git repository. |
| /// |
| /// This is the main entry for git clone/fetch. It does the followings: |
| /// |
| /// * Turns [`GitReference`] into refspecs accordingly. |
| /// * Dispatches `git fetch` using libgit2, gitoxide, or git CLI. |
| /// |
| /// The `remote_url` argument is the git remote URL where we want to fetch from. |
| /// |
| /// The `remote_kind` argument is a thing for [`-Zgitoxide`] shallow clones |
| /// at this time. It could be extended when libgit2 supports shallow clones. |
| /// |
| /// [`-Zgitoxide`]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#gitoxide |
| #[tracing::instrument(skip_all)] |
| pub fn fetch( |
| repo: &mut git2::Repository, |
| remote_url: &str, |
| manifest_reference: &GitReference, |
| locked_reference: &GitReference, |
| gctx: &GlobalContext, |
| remote_kind: RemoteKind, |
| ) -> CargoResult<()> { |
| if let Some(offline_flag) = gctx.offline_flag() { |
| anyhow::bail!( |
| "attempting to update a git repository, but {offline_flag} \ |
| was specified" |
| ) |
| } |
| |
| let shallow = remote_kind.to_shallow_setting(repo.is_shallow(), gctx); |
| |
| // Flag to keep track if the rev is a full commit hash |
| let mut fast_path_rev: bool = false; |
| |
| let oid_to_fetch = match github_fast_path(repo, remote_url, locked_reference, gctx) { |
| Ok(FastPathRev::UpToDate) => return Ok(()), |
| Ok(FastPathRev::NeedsFetch(rev)) => Some(rev), |
| Ok(FastPathRev::Indeterminate) => None, |
| Err(e) => { |
| debug!("failed to check github {:?}", e); |
| None |
| } |
| }; |
| |
| maybe_gc_repo(repo, gctx)?; |
| |
| clean_repo_temp_files(repo); |
| |
| // Translate the reference desired here into an actual list of refspecs |
| // which need to get fetched. Additionally record if we're fetching tags. |
| let mut refspecs = Vec::new(); |
| let mut tags = false; |
| // The `+` symbol on the refspec means to allow a forced (fast-forward) |
| // update which is needed if there is ever a force push that requires a |
| // fast-forward. |
| match locked_reference { |
| // For branches and tags we can fetch simply one reference and copy it |
| // locally, no need to fetch other branches/tags. |
| GitReference::Branch(b) => { |
| refspecs.push(format!("+refs/heads/{0}:refs/remotes/origin/{0}", b)); |
| } |
| |
| GitReference::Tag(t) => { |
| refspecs.push(format!("+refs/tags/{0}:refs/remotes/origin/tags/{0}", t)); |
| } |
| |
| GitReference::DefaultBranch => { |
| refspecs.push(String::from("+HEAD:refs/remotes/origin/HEAD")); |
| } |
| |
| GitReference::Rev(rev) => { |
| if rev.starts_with("refs/") { |
| refspecs.push(format!("+{0}:{0}", rev)); |
| } else if let Some(oid_to_fetch) = oid_to_fetch { |
| fast_path_rev = true; |
| refspecs.push(format!("+{0}:refs/commit/{0}", oid_to_fetch)); |
| } else if !matches!(shallow, gix::remote::fetch::Shallow::NoChange) |
| && rev_to_oid(rev).is_some() |
| { |
| // There is a specific commit to fetch and we will do so in shallow-mode only |
| // to not disturb the previous logic. |
| // Note that with typical settings for shallowing, we will just fetch a single `rev` |
| // as single commit. |
| // The reason we write to `refs/remotes/origin/HEAD` is that it's of special significance |
| // when during `GitReference::resolve()`, but otherwise it shouldn't matter. |
| refspecs.push(format!("+{0}:refs/remotes/origin/HEAD", rev)); |
| } else if let GitReference::Rev(rev) = manifest_reference |
| && rev.starts_with("refs/") |
| { |
| // If the lockfile has a commit. we can't directly fetch it (unless we're talking |
| // to GitHub), so we fetch the ref associated with it from the manifest. |
| refspecs.push(format!("+{0}:{0}", rev)); |
| } else { |
| // We don't know what the rev will point to. To handle this |
| // situation we fetch all branches and tags, and then we pray |
| // it's somewhere in there. |
| refspecs.push(String::from("+refs/heads/*:refs/remotes/origin/*")); |
| refspecs.push(String::from("+HEAD:refs/remotes/origin/HEAD")); |
| tags = true; |
| } |
| } |
| } |
| |
| debug!("doing a fetch for {remote_url}"); |
| let result = if let Some(true) = gctx.net_config()?.git_fetch_with_cli { |
| fetch_with_cli(repo, remote_url, &refspecs, tags, shallow, gctx) |
| } else if gctx.cli_unstable().gitoxide.map_or(false, |git| git.fetch) { |
| fetch_with_gitoxide(repo, remote_url, refspecs, tags, shallow, gctx) |
| } else { |
| fetch_with_libgit2(repo, remote_url, refspecs, tags, shallow, gctx) |
| }; |
| |
| if fast_path_rev { |
| if let Some(oid) = oid_to_fetch { |
| return result.with_context(|| format!("revision {} not found", oid)); |
| } |
| } |
| result |
| } |
| |
| /// `gitoxide` uses shallow locks to assure consistency when fetching to and to avoid races, and to write |
| /// files atomically. |
| /// Cargo has its own lock files and doesn't need that mechanism for race protection, so a stray lock means |
| /// a signal interrupted a previous shallow fetch and doesn't mean a race is happening. |
| fn has_shallow_lock_file(err: &crate::sources::git::fetch::Error) -> bool { |
| matches!( |
| err, |
| gix::env::collate::fetch::Error::Fetch(gix::remote::fetch::Error::Fetch( |
| gix::protocol::fetch::Error::LockShallowFile(_) |
| )) |
| ) |
| } |
| |
| /// Attempts to use `git` CLI installed on the system to fetch a repository, |
| /// when the config value [`net.git-fetch-with-cli`][1] is set. |
| /// |
| /// Unfortunately `libgit2` is notably lacking in the realm of authentication |
| /// when compared to the `git` command line. As a result, allow an escape |
| /// hatch for users that would prefer to use `git`-the-CLI for fetching |
| /// repositories instead of `libgit2`-the-library. This should make more |
| /// flavors of authentication possible while also still giving us all the |
| /// speed and portability of using `libgit2`. |
| /// |
| /// [1]: https://doc.rust-lang.org/nightly/cargo/reference/config.html#netgit-fetch-with-cli |
| #[tracing::instrument(skip(repo, gctx))] |
| fn fetch_with_cli( |
| repo: &mut git2::Repository, |
| url: &str, |
| refspecs: &[String], |
| tags: bool, |
| shallow: gix::remote::fetch::Shallow, |
| gctx: &GlobalContext, |
| ) -> CargoResult<()> { |
| debug!(target: "git-fetch", backend = "git-cli"); |
| |
| let mut cmd = ProcessBuilder::new("git"); |
| cmd.arg("fetch"); |
| if tags { |
| cmd.arg("--tags"); |
| } else { |
| cmd.arg("--no-tags"); |
| } |
| if let gix::remote::fetch::Shallow::DepthAtRemote(depth) = shallow { |
| let depth = 0i32.saturating_add_unsigned(depth.get()); |
| cmd.arg(format!("--depth={depth}")); |
| } |
| |
| let progress_config = gctx.progress_config(); |
| let progress = match progress_config.when { |
| ProgressWhen::Always => true, |
| ProgressWhen::Never => false, |
| ProgressWhen::Auto => { |
| // Recreate the same conditions used with `Progress` |
| let width = progress_config |
| .width |
| .or_else(|| gctx.shell().err_width().progress_max_width()); |
| gctx.shell().progress_supported() && width.is_some() |
| } |
| }; |
| if gctx.shell().verbosity() == Verbosity::Verbose { |
| cmd.arg("--verbose"); |
| } else if !progress { |
| cmd.arg("--quiet"); |
| } |
| |
| cmd.arg("--force") // handle force pushes |
| .arg("--update-head-ok") // see discussion in #2078 |
| .arg(url) |
| .args(refspecs) |
| // If cargo is run by git (for example, the `exec` command in `git |
| // rebase`), the GIT_DIR is set by git and will point to the wrong |
| // location. This makes sure GIT_DIR is always the repository path. |
| .env("GIT_DIR", repo.path()) |
| // The reset of these may not be necessary, but I'm including them |
| // just to be extra paranoid and avoid any issues. |
| .env_remove("GIT_WORK_TREE") |
| .env_remove("GIT_INDEX_FILE") |
| .env_remove("GIT_OBJECT_DIRECTORY") |
| .env_remove("GIT_ALTERNATE_OBJECT_DIRECTORIES") |
| .cwd(repo.path()); |
| gctx.shell() |
| .verbose(|s| s.status("Running", &cmd.to_string()))?; |
| network::retry::with_retry(gctx, || { |
| cmd.exec().map_err(|error| { |
| GitCliError::new(error) |
| .spurious(true) |
| .workaround( |
| "help: re-try with `net.git-fetch-with-cli = false` to see if it resolves the problem |
| https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli", |
| ) |
| .into() |
| }) |
| })?; |
| |
| Ok(()) |
| } |
| |
| #[tracing::instrument(skip(repo, gctx))] |
| fn fetch_with_gitoxide( |
| repo: &mut git2::Repository, |
| remote_url: &str, |
| refspecs: Vec<String>, |
| tags: bool, |
| shallow: gix::remote::fetch::Shallow, |
| gctx: &GlobalContext, |
| ) -> CargoResult<()> { |
| debug!(target: "git-fetch", backend = "gitoxide"); |
| |
| let git2_repo = repo; |
| let config_overrides = cargo_config_to_gitoxide_overrides(gctx)?; |
| let repo_reinitialized = AtomicBool::default(); |
| let res = oxide::with_retry_and_progress( |
| git2_repo.path(), |
| gctx, |
| remote_url, |
| &|repo_path, |
| should_interrupt, |
| mut progress, |
| url_for_authentication: &mut dyn FnMut(&gix::bstr::BStr)| { |
| // The `fetch` operation here may fail spuriously due to a corrupt |
| // repository. It could also fail, however, for a whole slew of other |
| // reasons (aka network related reasons). We want Cargo to automatically |
| // recover from corrupt repositories, but we don't want Cargo to stomp |
| // over other legitimate errors. |
| // |
| // Consequently we save off the error of the `fetch` operation and if it |
| // looks like a "corrupt repo" error then we blow away the repo and try |
| // again. If it looks like any other kind of error, or if we've already |
| // blown away the repository, then we want to return the error as-is. |
| loop { |
| let res = oxide::open_repo( |
| repo_path, |
| config_overrides.clone(), |
| oxide::OpenMode::ForFetch, |
| ) |
| .map_err(crate::sources::git::fetch::Error::from) |
| .and_then(|repo| { |
| debug!("initiating fetch of {refspecs:?} from {remote_url}"); |
| let url_for_authentication = &mut *url_for_authentication; |
| let remote = repo |
| .remote_at(remote_url)? |
| .with_fetch_tags(if tags { |
| gix::remote::fetch::Tags::All |
| } else { |
| gix::remote::fetch::Tags::Included |
| }) |
| .with_refspecs( |
| refspecs.iter().map(|s| s.as_str()), |
| gix::remote::Direction::Fetch, |
| ) |
| .map_err(crate::sources::git::fetch::Error::Other)?; |
| let url = remote |
| .url(gix::remote::Direction::Fetch) |
| .expect("set at init") |
| .to_owned(); |
| let connection = remote.connect(gix::remote::Direction::Fetch)?; |
| let mut authenticate = connection.configured_credentials(url)?; |
| let connection = connection.with_credentials( |
| move |action: gix::protocol::credentials::helper::Action| { |
| if let Some(url) = action |
| .context() |
| .and_then(|gctx| gctx.url.as_ref().filter(|url| *url != remote_url)) |
| { |
| url_for_authentication(url.as_ref()); |
| } |
| authenticate(action) |
| }, |
| ); |
| let outcome = connection |
| .prepare_fetch(&mut progress, gix::remote::ref_map::Options::default())? |
| .with_shallow(shallow.clone()) |
| .receive(&mut progress, should_interrupt)?; |
| Ok(outcome) |
| }); |
| let err = match res { |
| Ok(_) => break, |
| Err(e) => e, |
| }; |
| debug!("fetch failed: {}", err); |
| |
| if !repo_reinitialized.load(Ordering::Relaxed) |
| // We check for errors that could occur if the configuration, refs or odb files are corrupted. |
| // We don't check for errors related to writing as `gitoxide` is expected to create missing leading |
| // folder before writing files into it, or else not even open a directory as git repository (which is |
| // also handled here). |
| && err.is_corrupted() |
| || has_shallow_lock_file(&err) |
| { |
| repo_reinitialized.store(true, Ordering::Relaxed); |
| debug!( |
| "looks like this is a corrupt repository, reinitializing \ |
| and trying again" |
| ); |
| if oxide::reinitialize(repo_path).is_ok() { |
| continue; |
| } |
| } |
| |
| return Err(err.into()); |
| } |
| Ok(()) |
| }, |
| ); |
| if repo_reinitialized.load(Ordering::Relaxed) { |
| *git2_repo = git2::Repository::open(git2_repo.path())?; |
| } |
| res |
| } |
| |
| #[tracing::instrument(skip(repo, gctx))] |
| fn fetch_with_libgit2( |
| repo: &mut git2::Repository, |
| remote_url: &str, |
| refspecs: Vec<String>, |
| tags: bool, |
| shallow: gix::remote::fetch::Shallow, |
| gctx: &GlobalContext, |
| ) -> CargoResult<()> { |
| debug!(target: "git-fetch", backend = "libgit2"); |
| |
| static INIT: Once = Once::new(); |
| INIT.call_once(|| { |
| // SAFETY: |
| // |
| // The unsafety of the registration function derives from two aspects: |
| // |
| // 1. This call must be synchronized with all other registration calls as |
| // well as construction of new transports. |
| // 2. The argument is leaked. |
| // |
| // We're clear on point (1) because this is the only `register` call we make in the |
| // cargo-binary and cargo-library is not officially supported and it is under a lock. |
| // Technically, `git2_curl` also has a lock but that isn't part of their API guarantees. |
| // |
| // We're mostly clear on point (2) because we'd only free it after everything is done anyway |
| unsafe { init_git_transports(gctx) }; |
| }); |
| |
| let git_config = git2::Config::open_default()?; |
| with_fetch_options(&git_config, remote_url, gctx, &mut |mut opts| { |
| if tags { |
| opts.download_tags(git2::AutotagOption::All); |
| } |
| if let gix::remote::fetch::Shallow::DepthAtRemote(depth) = shallow { |
| opts.depth(0i32.saturating_add_unsigned(depth.get())); |
| } |
| // The `fetch` operation here may fail spuriously due to a corrupt |
| // repository. It could also fail, however, for a whole slew of other |
| // reasons (aka network related reasons). We want Cargo to automatically |
| // recover from corrupt repositories, but we don't want Cargo to stomp |
| // over other legitimate errors. |
| // |
| // Consequently we save off the error of the `fetch` operation and if it |
| // looks like a "corrupt repo" error then we blow away the repo and try |
| // again. If it looks like any other kind of error, or if we've already |
| // blown away the repository, then we want to return the error as-is. |
| let mut repo_reinitialized = false; |
| loop { |
| debug!("initiating fetch of {refspecs:?} from {remote_url}"); |
| let res = repo |
| .remote_anonymous(remote_url)? |
| .fetch(&refspecs, Some(&mut opts), None); |
| let err = match res { |
| Ok(()) => break, |
| Err(e) => e, |
| }; |
| debug!("fetch failed: {}", err); |
| |
| if !repo_reinitialized && matches!(err.class(), ErrorClass::Reference | ErrorClass::Odb) |
| { |
| repo_reinitialized = true; |
| debug!( |
| "looks like this is a corrupt repository, reinitializing \ |
| and trying again" |
| ); |
| if reinitialize(repo).is_ok() { |
| continue; |
| } |
| } |
| |
| return Err(err.into()); |
| } |
| Ok(()) |
| }) |
| } |
| |
| /// Configure libgit2 to use libcurl if necessary. |
| /// |
| /// If the user has a non-default network configuration, then libgit2 will be |
| /// configured to use libcurl instead of the built-in networking support so |
| /// that those configuration settings can be used. |
| /// |
| /// # Safety |
| /// |
| /// See [git2_curl::register] |
| #[tracing::instrument(skip_all)] |
| unsafe fn init_git_transports(gctx: &GlobalContext) { |
| match needs_custom_http_transport(gctx) { |
| Ok(true) => {} |
| _ => return, |
| } |
| |
| let handle = match http_handle(gctx) { |
| Ok(handle) => handle, |
| Err(..) => return, |
| }; |
| |
| unsafe { |
| git2_curl::register(handle); |
| } |
| } |
| |
| /// Attempts to `git gc` a repository. |
| /// |
| /// Cargo has a bunch of long-lived git repositories in its global cache and |
| /// some, like the index, are updated very frequently. Right now each update |
| /// creates a new "pack file" inside the git database, and over time this can |
| /// cause bad performance and bad current behavior in libgit2. |
| /// |
| /// One pathological use case today is where libgit2 opens hundreds of file |
| /// descriptors, getting us dangerously close to blowing out the OS limits of |
| /// how many fds we can have open. This is detailed in [#4403]. |
| /// |
| /// Instead of trying to be clever about when gc is needed, we just run |
| /// `git gc --auto` and let git figure it out. It checks its own thresholds |
| /// (gc.auto, gc.autoPackLimit) and either does the work or exits quickly. |
| /// If git isn't installed, no worries - we skip it. |
| /// |
| /// [#4403]: https://github.com/rust-lang/cargo/issues/4403 |
| fn maybe_gc_repo(repo: &mut git2::Repository, gctx: &GlobalContext) -> CargoResult<()> { |
| // Let git decide whether gc is actually needed based on its own thresholds |
| // (gc.auto, gc.autoPackLimit). This avoids duplicating git's internal logic |
| // for deciding when housekeeping is needed. |
| // |
| // For testing purposes, __CARGO_PACKFILE_LIMIT can be set to override |
| // gc.autoPackLimit, which has the same meaning. This lets tests force gc |
| // to run by setting a low threshold without depending on git's defaults. |
| let mut cmd = Command::new("git"); |
| if let Ok(limit) = gctx.get_env("__CARGO_PACKFILE_LIMIT") { |
| cmd.arg(format!("-c gc.autoPackLimit={}", limit)); |
| } |
| cmd.arg("gc").arg("--auto").current_dir(repo.path()); |
| |
| match cmd.output() { |
| Ok(out) => { |
| debug!( |
| "git-gc --auto status: {}\n\nstdout ---\n{}\nstderr ---\n{}", |
| out.status, |
| String::from_utf8_lossy(&out.stdout), |
| String::from_utf8_lossy(&out.stderr) |
| ); |
| if out.status.success() { |
| let new = git2::Repository::open(repo.path())?; |
| *repo = new; |
| return Ok(()); |
| } |
| } |
| Err(e) => debug!("git-gc --auto failed to spawn: {}", e), |
| } |
| |
| // Alright all else failed, let's start over. |
| reinitialize(repo) |
| } |
| |
| /// Removes temporary files left from previous activity. |
| /// |
| /// If libgit2 is interrupted while indexing pack files, it will leave behind |
| /// some temporary files that it doesn't clean up. These can be quite large in |
| /// size, so this tries to clean things up. |
| /// |
| /// This intentionally ignores errors. This is only an opportunistic cleaning, |
| /// and we don't really care if there are issues (there's unlikely anything |
| /// that can be done). |
| /// |
| /// The git CLI has similar behavior (its temp files look like |
| /// `objects/pack/tmp_pack_9kUSA8`). Those files are normally deleted via `git |
| /// prune` which is run by `git gc`. However, it doesn't know about libgit2's |
| /// filenames, so they never get cleaned up. |
| fn clean_repo_temp_files(repo: &git2::Repository) { |
| let path = repo.path().join("objects/pack/pack_git2_*"); |
| let Some(pattern) = path.to_str() else { |
| tracing::warn!("cannot convert {path:?} to a string"); |
| return; |
| }; |
| let Ok(paths) = glob::glob(pattern) else { |
| return; |
| }; |
| for path in paths { |
| if let Ok(path) = path { |
| match paths::remove_file(&path) { |
| Ok(_) => tracing::debug!("removed stale temp git file {path:?}"), |
| Err(e) => { |
| tracing::warn!("failed to remove {path:?} while cleaning temp files: {e}") |
| } |
| } |
| } |
| } |
| } |
| |
| /// Reinitializes a given Git repository. This is useful when a Git repository |
| /// seems corrupted and we want to start over. |
| fn reinitialize(repo: &mut git2::Repository) -> CargoResult<()> { |
| // Here we want to drop the current repository object pointed to by `repo`, |
| // so we initialize temporary repository in a sub-folder, blow away the |
| // existing git folder, and then recreate the git repo. Finally we blow away |
| // the `tmp` folder we allocated. |
| let path = repo.path().to_path_buf(); |
| debug!("reinitializing git repo at {:?}", path); |
| let tmp = path.join("tmp"); |
| let bare = !repo.path().ends_with(".git"); |
| *repo = init(&tmp, false)?; |
| for entry in path.read_dir()? { |
| let entry = entry?; |
| if entry.file_name().to_str() == Some("tmp") { |
| continue; |
| } |
| let path = entry.path(); |
| drop(paths::remove_file(&path).or_else(|_| paths::remove_dir_all(&path))); |
| } |
| *repo = init(&path, bare)?; |
| paths::remove_dir_all(&tmp)?; |
| Ok(()) |
| } |
| |
| /// Initializes a Git repository at `path`. |
| fn init(path: &Path, bare: bool) -> CargoResult<git2::Repository> { |
| let mut opts = git2::RepositoryInitOptions::new(); |
| // Skip anything related to templates, they just call all sorts of issues as |
| // we really don't want to use them yet they insist on being used. See #6240 |
| // for an example issue that comes up. |
| opts.external_template(false); |
| opts.bare(bare); |
| Ok(git2::Repository::init_opts(&path, &opts)?) |
| } |
| |
| /// The result of GitHub fast path check. See [`github_fast_path`] for more. |
| enum FastPathRev { |
| /// The local rev (determined by `reference.resolve(repo)`) is already up to |
| /// date with what this rev resolves to on GitHub's server. |
| UpToDate, |
| /// The following SHA must be fetched in order for the local rev to become |
| /// up to date. |
| NeedsFetch(Oid), |
| /// Don't know whether local rev is up to date. We'll fetch _all_ branches |
| /// and tags from the server and see what happens. |
| Indeterminate, |
| } |
| |
| /// Attempts GitHub's special fast path for testing if we've already got an |
| /// up-to-date copy of the repository. |
| /// |
| /// Updating the index is done pretty regularly so we want it to be as fast as |
| /// possible. For registries hosted on GitHub (like the crates.io index) there's |
| /// a fast path available to use[^1] to tell us that there's no updates to be |
| /// made. |
| /// |
| /// Note that this function should never cause an actual failure because it's |
| /// just a fast path. As a result, a caller should ignore `Err` returned from |
| /// this function and move forward on the normal path. |
| /// |
| /// [^1]: <https://developer.github.com/v3/repos/commits/#get-the-sha-1-of-a-commit-reference> |
| #[tracing::instrument(skip(repo, gctx))] |
| fn github_fast_path( |
| repo: &mut git2::Repository, |
| url: &str, |
| reference: &GitReference, |
| gctx: &GlobalContext, |
| ) -> CargoResult<FastPathRev> { |
| let url = Url::parse(url)?; |
| if !is_github(&url) { |
| return Ok(FastPathRev::Indeterminate); |
| } |
| |
| let local_object = resolve_ref(reference, repo).ok(); |
| |
| let github_branch_name = match reference { |
| GitReference::Branch(branch) => branch, |
| GitReference::Tag(tag) => tag, |
| GitReference::DefaultBranch => "HEAD", |
| GitReference::Rev(rev) => { |
| if rev.starts_with("refs/") { |
| rev |
| } else if looks_like_commit_hash(rev) { |
| // `revparse_single` (used by `resolve`) is the only way to turn |
| // short hash -> long hash, but it also parses other things, |
| // like branch and tag names, which might coincidentally be |
| // valid hex. |
| // |
| // We only return early if `rev` is a prefix of the object found |
| // by `revparse_single`. Don't bother talking to GitHub in that |
| // case, since commit hashes are permanent. If a commit with the |
| // requested hash is already present in the local clone, its |
| // contents must be the same as what is on the server for that |
| // hash. |
| // |
| // If `rev` is not found locally by `revparse_single`, we'll |
| // need GitHub to resolve it and get a hash. If `rev` is found |
| // but is not a short hash of the found object, it's probably a |
| // branch and we also need to get a hash from GitHub, in case |
| // the branch has moved. |
| if let Some(local_object) = local_object { |
| if is_short_hash_of(rev, local_object) { |
| debug!("github fast path already has {local_object}"); |
| return Ok(FastPathRev::UpToDate); |
| } |
| } |
| // If `rev` is a full commit hash, the only thing it can resolve |
| // to is itself. Don't bother talking to GitHub in that case |
| // either. (This ensures that we always attempt to fetch the |
| // commit directly even if we can't reach the GitHub API.) |
| if let Some(oid) = rev_to_oid(rev) { |
| debug!("github fast path is already a full commit hash {rev}"); |
| return Ok(FastPathRev::NeedsFetch(oid)); |
| } |
| rev |
| } else { |
| debug!("can't use github fast path with `rev = \"{}\"`", rev); |
| return Ok(FastPathRev::Indeterminate); |
| } |
| } |
| }; |
| |
| // This expects GitHub urls in the form `github.com/user/repo` and nothing |
| // else |
| let mut pieces = url |
| .path_segments() |
| .ok_or_else(|| anyhow!("no path segments on url"))?; |
| let username = pieces |
| .next() |
| .ok_or_else(|| anyhow!("couldn't find username"))?; |
| let repository = pieces |
| .next() |
| .ok_or_else(|| anyhow!("couldn't find repository name"))?; |
| if pieces.next().is_some() { |
| anyhow::bail!("too many segments on URL"); |
| } |
| |
| // Trim off the `.git` from the repository, if present, since that's |
| // optional for GitHub and won't work when we try to use the API as well. |
| let repository = repository.strip_suffix(".git").unwrap_or(repository); |
| |
| let url = format!( |
| "https://api.github.com/repos/{}/{}/commits/{}", |
| username, repository, github_branch_name, |
| ); |
| debug!("attempting GitHub fast path for {}", url); |
| let mut request = |
| Request::get(url).header(http::header::ACCEPT, "application/vnd.github.3.sha"); |
| if let Some(local_object) = local_object { |
| request = request.header(http::header::IF_NONE_MATCH, &format!("\"{local_object}\"")); |
| } |
| let response = gctx |
| .http_async()? |
| .request_blocking(request.body(Vec::new())?)?; |
| let response_code = response.status(); |
| if response_code == StatusCode::NOT_MODIFIED { |
| debug!("github fast path up-to-date"); |
| Ok(FastPathRev::UpToDate) |
| } else if response_code == StatusCode::OK |
| && let Some(oid_to_fetch) = rev_to_oid(str::from_utf8(&response.body())?) |
| { |
| // response expected to be a full hash hexstring (40 or 64 chars) |
| debug!("github fast path fetch {oid_to_fetch}"); |
| Ok(FastPathRev::NeedsFetch(oid_to_fetch)) |
| } else { |
| // Usually response_code == 404 if the repository does not exist, and |
| // response_code == 422 if exists but GitHub is unable to resolve the |
| // requested rev. |
| debug!("github fast path bad response code {response_code}"); |
| Ok(FastPathRev::Indeterminate) |
| } |
| } |
| |
| /// Whether a `url` is one from GitHub. |
| fn is_github(url: &Url) -> bool { |
| url.host_str() == Some("github.com") |
| } |
| |
| // Give some messages on GitHub PR URL given as is |
| pub(crate) fn note_github_pull_request(url: &str) -> Option<String> { |
| if let Ok(url) = url.parse::<Url>() |
| && is_github(&url) |
| { |
| let path_segments = url |
| .path_segments() |
| .map(|p| p.into_iter().collect::<Vec<_>>()) |
| .unwrap_or_default(); |
| if let [owner, repo, "pull", pr_number, ..] = path_segments[..] { |
| let repo_url = format!("https://github.com/{owner}/{repo}.git"); |
| let rev = format!("refs/pull/{pr_number}/head"); |
| return Some(format!( |
| concat!( |
| "\n\nnote: GitHub url {} is not a repository. \n", |
| "help: Replace the dependency with \n", |
| " `git = \"{}\" rev = \"{}\"` \n", |
| " to specify pull requests as dependencies' revision." |
| ), |
| url, repo_url, rev |
| )); |
| } |
| } |
| |
| None |
| } |
| |
| /// Whether a `rev` looks like a commit hash (ASCII hex digits). |
| fn looks_like_commit_hash(rev: &str) -> bool { |
| rev.len() >= 7 && rev.chars().all(|ch| ch.is_ascii_hexdigit()) |
| } |
| |
| /// Whether `rev` is a shorter hash of `oid`. |
| fn is_short_hash_of(rev: &str, oid: Oid) -> bool { |
| let long_hash = oid.to_string(); |
| match long_hash.get(..rev.len()) { |
| Some(truncated_long_hash) => truncated_long_hash.eq_ignore_ascii_case(rev), |
| None => false, |
| } |
| } |
| |
| #[cfg(test)] |
| mod tests { |
| use super::*; |
| |
| #[test] |
| fn github_fast_path_full_hash_returns_needs_fetch() { |
| let temp_dir = tempfile::TempDir::new().unwrap(); |
| let mut repo = git2::Repository::init_bare(temp_dir.path()).unwrap(); |
| let full_hash = "c9040898c9183ddbb9402dcbf749ed06d6ea90ad"; |
| let reference = GitReference::Rev(full_hash.to_string()); |
| let gctx = GlobalContext::default().unwrap(); |
| let expected_oid = rev_to_oid(full_hash).unwrap(); |
| |
| let result = |
| github_fast_path(&mut repo, "https://github.com/user/repo", &reference, &gctx).unwrap(); |
| |
| assert!(matches!(result, FastPathRev::NeedsFetch(oid) if oid == expected_oid)); |
| } |
| |
| #[test] |
| fn test_absolute_submodule_url() { |
| let cases = [ |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo", |
| "git@github.com:rust-lang/cargo.git", |
| "git@github.com:rust-lang/cargo.git", |
| ), |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo", |
| "./", |
| "ssh://git@gitub.com/rust-lang/cargo/", |
| ), |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo", |
| "../", |
| "ssh://git@gitub.com/rust-lang/", |
| ), |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo", |
| "./foo", |
| "ssh://git@gitub.com/rust-lang/cargo/foo", |
| ), |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo/", |
| "./foo", |
| "ssh://git@gitub.com/rust-lang/cargo/foo", |
| ), |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo/", |
| "../foo", |
| "ssh://git@gitub.com/rust-lang/foo", |
| ), |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo", |
| "../foo", |
| "ssh://git@gitub.com/rust-lang/foo", |
| ), |
| ( |
| "ssh://git@gitub.com/rust-lang/cargo", |
| "../foo/bar/../baz", |
| "ssh://git@gitub.com/rust-lang/foo/baz", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git", |
| "ssh://git@gitub.com/rust-lang/cargo", |
| "ssh://git@gitub.com/rust-lang/cargo", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git", |
| "./", |
| "git@github.com:rust-lang/cargo.git/./", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git", |
| "../", |
| "git@github.com:rust-lang/cargo.git/../", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git", |
| "./foo", |
| "git@github.com:rust-lang/cargo.git/./foo", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git/", |
| "./foo", |
| "git@github.com:rust-lang/cargo.git/./foo", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git", |
| "../foo", |
| "git@github.com:rust-lang/cargo.git/../foo", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git/", |
| "../foo", |
| "git@github.com:rust-lang/cargo.git/../foo", |
| ), |
| ( |
| "git@github.com:rust-lang/cargo.git", |
| "../foo/bar/../baz", |
| "git@github.com:rust-lang/cargo.git/../foo/bar/../baz", |
| ), |
| ]; |
| |
| for (base_url, submodule_url, expected) in cases { |
| let url = absolute_submodule_url(base_url, submodule_url).unwrap(); |
| assert_eq!( |
| expected, url, |
| "base `{base_url}`; submodule `{submodule_url}`" |
| ); |
| } |
| } |
| } |
| |
| /// Turns a full commit hash revision into an oid. |
| /// |
| /// Git object ID is supposed to be a hex string of 20 (SHA1) or 32 (SHA256) bytes. |
| /// Its length must be double to the underlying bytes (40 or 64), |
| /// otherwise libgit2 would happily zero-pad the returned oid. |
| /// |
| /// See: |
| /// |
| /// * <https://github.com/rust-lang/cargo/issues/13188> |
| /// * <https://github.com/rust-lang/cargo/issues/13968> |
| pub(super) fn rev_to_oid(rev: &str) -> Option<Oid> { |
| Oid::from_str(rev) |
| .ok() |
| .filter(|oid| oid.as_bytes().len() * 2 == rev.len()) |
| } |
