| //! Access to a Git index based registry. See [`GitRegistry`] for details. |
| |
| use crate::sources::git; |
| use crate::sources::git::fetch::RemoteKind; |
| use crate::sources::git::resolve_ref; |
| use crate::sources::registry::MaybeLock; |
| use crate::sources::registry::download; |
| use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData}; |
| use crate::util::cache_lock::CacheLockMode; |
| use crate::util::errors::CargoResult; |
| use crate::util::interning::InternedString; |
| use crate::util::{Filesystem, GlobalContext}; |
| use crate::workspace::global_cache_tracker; |
| use crate::workspace::{GitReference, PackageId, SourceId}; |
| use anyhow::Context as _; |
| use cargo_util::paths; |
| use std::cell::{Cell, Ref, RefCell}; |
| use std::fs::File; |
| use std::mem; |
| use std::path::Path; |
| use std::str; |
| use tracing::{debug, trace}; |
| |
| /// A remote registry is a registry that lives at a remote URL (such as |
| /// crates.io). The git index is cloned locally, and `.crate` files are |
| /// downloaded as needed and cached locally. |
| /// |
| /// This type is primarily accessed through the [`RegistryData`] trait. |
| /// |
| /// See the [module-level documentation](super) for the index format and layout. |
| /// |
| /// ## History of Git-based index registry |
| /// |
| /// Using Git to host this index used to be quite efficient. The full index can |
| /// be stored efficiently locally on disk, and once it is downloaded, all |
| /// queries of a registry can happen locally and needn't touch the network. |
| /// Git-based index was a reasonable design choice at the time when HTTP/2 |
| /// was just introduced. |
| /// |
| /// However, the full index keeps growing as crates.io grows. It becomes |
| /// relatively big and slows down the first use of Cargo. Git (specifically |
| /// libgit2) is not efficient at handling huge amounts of small files either. |
| /// On the other hand, newer protocols like HTTP/2 are prevalent and capable to |
| /// serve a bunch of tiny files. Today, it is encouraged to use [`HttpRegistry`], |
| /// which is the default from 1.70.0. That being said, Cargo will continue |
| /// supporting Git-based index for a pretty long while. |
| /// |
| /// [`HttpRegistry`]: super::http_remote::HttpRegistry |
| pub struct GitRegistry<'gctx> { |
| /// The name of this source, a unique string (across all sources) used as |
| /// the directory name where its cached content is stored. |
| name: InternedString, |
| /// Path to the registry index (`$CARGO_HOME/registry/index/$REG-HASH`). |
| index_path: Filesystem, |
| /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/cache/$REG-HASH`). |
| cache_path: Filesystem, |
| /// The unique identifier of this registry source. |
| source_id: SourceId, |
| /// This reference is stored so that when a registry needs update, it knows |
| /// where to fetch from. |
| index_git_ref: GitReference, |
| gctx: &'gctx GlobalContext, |
| /// A Git [tree object] to help this registry find crate metadata from the |
| /// underlying Git repository. |
| /// |
| /// This is stored here to prevent Git from repeatedly creating a tree object |
| /// during each call into `load()`. |
| /// |
| /// [tree object]: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects#_tree_objects |
| tree: RefCell<Option<git2::Tree<'static>>>, |
| /// A Git repository that contains the actual index we want. |
| repo: RefCell<Option<git2::Repository>>, |
| /// The current HEAD commit of the underlying Git repository. |
| head: Cell<Option<git2::Oid>>, |
| /// This stores sha value of the current HEAD commit for convenience. |
| current_sha: Cell<Option<InternedString>>, |
| /// Whether this registry needs to update package information. |
| /// |
| /// See [`GitRegistry::mark_updated`] on how to make sure a registry |
| /// index is updated only once per session. |
| needs_update: Cell<bool>, |
| /// Disables status messages. |
| quiet: bool, |
| } |
| |
| impl<'gctx> GitRegistry<'gctx> { |
| /// Creates a Git-rebased remote registry for `source_id`. |
| /// |
| /// * `name` --- Name of a path segment where `.crate` tarballs and the |
| /// registry index are stored. Expect to be unique. |
| pub fn new(source_id: SourceId, gctx: &'gctx GlobalContext, name: &str) -> GitRegistry<'gctx> { |
| GitRegistry { |
| name: name.into(), |
| index_path: gctx.registry_index_path().join(name), |
| cache_path: gctx.registry_cache_path().join(name), |
| source_id, |
| gctx, |
| index_git_ref: GitReference::DefaultBranch, |
| tree: RefCell::new(None), |
| repo: RefCell::new(None), |
| head: Cell::new(None), |
| current_sha: Cell::new(None), |
| needs_update: Cell::new(false), |
| quiet: false, |
| } |
| } |
| |
| /// Creates intermediate dirs and initialize the repository. |
| fn repo(&self) -> CargoResult<Ref<'_, Option<git2::Repository>>> { |
| if self.repo.borrow().is_none() { |
| trace!("acquiring registry index lock"); |
| let path = self |
| .gctx |
| .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path); |
| |
| self.repo.replace(Some(match git2::Repository::open(&path) { |
| Ok(repo) => repo, |
| Err(_) => { |
| drop(paths::remove_dir_all(&path)); |
| paths::create_dir_all(&path)?; |
| |
| // Note that we'd actually prefer to use a bare repository |
| // here as we're not actually going to check anything out. |
| // All versions of Cargo, though, share the same CARGO_HOME, |
| // so for compatibility with older Cargo which *does* do |
| // checkouts we make sure to initialize a new full |
| // repository (not a bare one). |
| // |
| // We should change this to `init_bare` whenever we feel |
| // like enough time has passed or if we change the directory |
| // that the folder is located in, such as by changing the |
| // hash at the end of the directory. |
| // |
| // Note that in the meantime we also skip `init.templatedir` |
| // as it can be misconfigured sometimes or otherwise add |
| // things that we don't want. |
| let mut opts = git2::RepositoryInitOptions::new(); |
| opts.external_template(false); |
| git2::Repository::init_opts(&path, &opts).with_context(|| { |
| format!("failed to initialize index git repository (in {:?})", path) |
| })? |
| } |
| })); |
| } |
| |
| Ok(self.repo.borrow()) |
| } |
| |
| /// Get the object ID of the HEAD commit from the underlying Git repository. |
| fn head(&self) -> CargoResult<git2::Oid> { |
| if self.head.get().is_none() { |
| let repo = self.repo()?; |
| let repo = repo.as_ref().unwrap(); |
| let oid = resolve_ref(&self.index_git_ref, repo)?; |
| self.head.set(Some(oid)); |
| } |
| Ok(self.head.get().unwrap()) |
| } |
| |
| /// Returns a [`git2::Tree`] object of the current HEAD commit of the |
| /// underlying Git repository. |
| fn tree(&self) -> CargoResult<Ref<'_, git2::Tree<'_>>> { |
| { |
| let tree = self.tree.borrow(); |
| if tree.is_some() { |
| return Ok(Ref::map(tree, |s| s.as_ref().unwrap())); |
| } |
| } |
| let repo = self.repo()?; |
| let repo = repo.as_ref().unwrap(); |
| let commit = repo.find_commit(self.head()?)?; |
| let tree = commit.tree()?; |
| |
| // SAFETY: |
| // Unfortunately in libgit2 the tree objects look like they've got a |
| // reference to the repository object which means that a tree cannot |
| // outlive the repository that it came from. Here we want to cache this |
| // tree, though, so to accomplish this we transmute it to a static |
| // lifetime. |
| // |
| // Note that we don't actually hand out the static lifetime, instead we |
| // only return a scoped one from this function. Additionally the repo |
| // we loaded from (above) lives as long as this object |
| // (`GitRegistry`) so we then just need to ensure that the tree is |
| // destroyed first in the destructor, hence the destructor on |
| // `GitRegistry` below. |
| let tree = unsafe { mem::transmute::<git2::Tree<'_>, git2::Tree<'static>>(tree) }; |
| *self.tree.borrow_mut() = Some(tree); |
| Ok(Ref::map(self.tree.borrow(), |s| s.as_ref().unwrap())) |
| } |
| |
| /// Gets the current version of the registry index. |
| /// |
| /// It is usually sha of the HEAD commit from the underlying Git repository. |
| fn current_version(&self) -> Option<InternedString> { |
| if let Some(sha) = self.current_sha.get() { |
| return Some(sha); |
| } |
| let sha = self.head().ok()?.to_string().into(); |
| self.current_sha.set(Some(sha)); |
| Some(sha) |
| } |
| |
| /// Whether the registry is up-to-date. See [`Self::mark_updated`] for more. |
| fn is_updated(&self) -> bool { |
| self.gctx.updated_sources().contains(&self.source_id) |
| } |
| |
| /// Marks this registry as up-to-date. |
| /// |
| /// This makes sure the index is only updated once per session since it is |
| /// an expensive operation. This generally only happens when the resolver |
| /// is run multiple times, such as during `cargo publish`. |
| fn mark_updated(&self) { |
| self.gctx.updated_sources().insert(self.source_id); |
| } |
| |
| fn update(&self) -> CargoResult<()> { |
| if !self.needs_update.get() { |
| return Ok(()); |
| } |
| |
| self.needs_update.set(false); |
| |
| if self.is_updated() { |
| return Ok(()); |
| } |
| self.mark_updated(); |
| |
| if !self.gctx.network_allowed() { |
| return Ok(()); |
| } |
| if self.gctx.cli_unstable().no_index_update { |
| return Ok(()); |
| } |
| |
| debug!("updating the index"); |
| |
| // Ensure that we'll actually be able to acquire an HTTP handle later on |
| // once we start trying to download crates. This will weed out any |
| // problems with `.cargo/config` configuration related to HTTP. |
| // |
| // This way if there's a problem the error gets printed before we even |
| // hit the index, which may not actually read this configuration. |
| self.gctx.http()?; |
| |
| self.prepare()?; |
| self.head.set(None); |
| *self.tree.borrow_mut() = None; |
| self.current_sha.set(None); |
| let _path = self |
| .gctx |
| .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path); |
| if !self.quiet { |
| self.gctx |
| .shell() |
| .status("Updating", self.source_id.display_index())?; |
| } |
| |
| // Fetch the latest version of our `index_git_ref` into the index |
| // checkout. |
| let url = self.source_id.url(); |
| let mut repo = self.repo.borrow_mut(); |
| let repo = repo.as_mut().unwrap(); |
| git::fetch( |
| repo, |
| url.as_str(), |
| &self.index_git_ref, |
| &self.index_git_ref, |
| self.gctx, |
| RemoteKind::Registry, |
| ) |
| .with_context(|| format!("failed to fetch `{}`", url))?; |
| |
| Ok(()) |
| } |
| } |
| |
| #[async_trait::async_trait(?Send)] |
| impl<'gctx> RegistryData for GitRegistry<'gctx> { |
| fn prepare(&self) -> CargoResult<()> { |
| self.repo()?; |
| self.gctx |
| .deferred_global_last_use()? |
| .mark_registry_index_used(global_cache_tracker::RegistryIndex { |
| encoded_registry_name: self.name, |
| }); |
| Ok(()) |
| } |
| |
| fn index_path(&self) -> &Filesystem { |
| &self.index_path |
| } |
| |
| fn cache_path(&self) -> &Filesystem { |
| &self.cache_path |
| } |
| |
| fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path { |
| self.gctx |
| .assert_package_cache_locked(CacheLockMode::DownloadExclusive, path) |
| } |
| |
| /// Read the general concept for `load()` on [`RegistryData::load`]. |
| /// |
| /// `index_version` is a string representing the version of the file used |
| /// to construct the cached copy. |
| /// |
| /// Older versions of Cargo used the single value of the hash of the HEAD |
| /// commit as a `index_version`. This is technically correct but a little |
| /// too conservative. If a new commit is fetched all cached files need to |
| /// be regenerated even if a particular file was not changed. |
| /// |
| /// However if an old cargo has written such a file we still know how to |
| /// read it, as long as we check for that hash value. |
| /// |
| /// Cargo now uses a hash of the file's contents as provided by git. |
| async fn load( |
| &self, |
| _root: &Path, |
| path: &Path, |
| index_version: Option<&str>, |
| ) -> CargoResult<LoadResponse> { |
| if self.needs_update.get() { |
| self.update()?; |
| } |
| // Check if the cache is valid. |
| let git_commit_hash = self.current_version(); |
| if index_version.is_some() && index_version == git_commit_hash.as_deref() { |
| // This file was written by an old version of cargo, but it is |
| // still up-to-date. |
| return Ok(LoadResponse::CacheValid); |
| } |
| // Note that the index calls this method and the filesystem is locked |
| // in the index, so we don't need to worry about an `update_index` |
| // happening in a different process. |
| fn load_helper( |
| registry: &GitRegistry<'_>, |
| path: &Path, |
| index_version: Option<&str>, |
| ) -> CargoResult<LoadResponse> { |
| let repo = registry.repo()?; |
| let repo = repo.as_ref().unwrap(); |
| let tree = registry.tree()?; |
| let entry = tree.get_path(path); |
| let entry = entry?; |
| let git_file_hash = Some(entry.id().to_string()); |
| |
| // Check if the cache is valid. |
| if index_version.is_some() && index_version == git_file_hash.as_deref() { |
| return Ok(LoadResponse::CacheValid); |
| } |
| |
| let object = entry.to_object(repo)?; |
| let Some(blob) = object.as_blob() else { |
| anyhow::bail!("path `{}` is not a blob in the git repo", path.display()) |
| }; |
| |
| Ok(LoadResponse::Data { |
| raw_data: blob.content().to_vec(), |
| index_version: git_file_hash, |
| }) |
| } |
| |
| loop { |
| return match load_helper(&self, path, index_version) { |
| Ok(result) => Ok(result), |
| Err(_) if !self.is_updated() => { |
| // If git returns an error and we haven't updated the repo, |
| // return pending to allow an update to try again. |
| self.needs_update.set(true); |
| self.update()?; |
| continue; |
| } |
| Err(e) |
| if e.downcast_ref::<git2::Error>() |
| .map(|e| e.code() == git2::ErrorCode::NotFound) |
| .unwrap_or_default() => |
| { |
| // The repo has been updated and the file does not exist. |
| Ok(LoadResponse::NotFound) |
| } |
| Err(e) => Err(e), |
| }; |
| } |
| } |
| |
| async fn config(&self) -> CargoResult<Option<RegistryConfig>> { |
| debug!("loading config"); |
| self.prepare()?; |
| self.gctx |
| .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path); |
| match self |
| .load(Path::new(""), Path::new(RegistryConfig::NAME), None) |
| .await? |
| { |
| LoadResponse::Data { raw_data, .. } => { |
| trace!("config loaded"); |
| let cfg: RegistryConfig = serde_json::from_slice(&raw_data)?; |
| Ok(Some(cfg)) |
| } |
| _ => Ok(None), |
| } |
| } |
| |
| /// Read the general concept for `invalidate_cache()` on |
| /// [`RegistryData::invalidate_cache`]. |
| /// |
| /// To fully invalidate, undo [`GitRegistry::mark_updated`]'s work. |
| fn invalidate_cache(&self) { |
| self.needs_update.set(true); |
| } |
| |
| fn set_quiet(&mut self, quiet: bool) { |
| self.quiet = quiet; |
| } |
| |
| fn is_updated(&self) -> bool { |
| self.is_updated() |
| } |
| |
| async fn download(&self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock> { |
| let registry_config = self.config().await?.unwrap(); |
| |
| download::download( |
| &self.cache_path, |
| &self.gctx, |
| self.name, |
| pkg, |
| checksum, |
| registry_config, |
| ) |
| } |
| |
| async fn finish_download( |
| &self, |
| pkg: PackageId, |
| checksum: &str, |
| data: &[u8], |
| ) -> CargoResult<File> { |
| download::finish_download( |
| &self.cache_path, |
| &self.gctx, |
| self.name.clone(), |
| pkg, |
| checksum, |
| data, |
| ) |
| } |
| |
| fn is_crate_downloaded(&self, pkg: PackageId) -> bool { |
| download::is_crate_downloaded(&self.cache_path, &self.gctx, pkg) |
| } |
| } |
| |
| /// Implemented to just be sure to drop `tree` field before our other fields. |
| /// See SAFETY inside [`GitRegistry::tree()`] for more. |
| impl<'gctx> Drop for GitRegistry<'gctx> { |
| fn drop(&mut self) { |
| self.tree.borrow_mut().take(); |
| } |
| } |