Add a nonstandard shallow clone for GitHub

Author: Eh2406

src/cargo/core/source/source_id.rs

@@ -4,8 +4,7 @@ use crate::sources::{DirectorySource, CRATES_IO_DOMAIN, CRATES_IO_INDEX, CRATES_
 use crate::sources::{GitSource, PathSource, RegistrySource};
 use crate::util::{CanonicalUrl, CargoResult, Config, IntoUrl};
 use log::trace;
-use serde::de;
-use serde::ser;
+use serde::{de, ser, Serialize};
 use std::cmp::{self, Ordering};
 use std::collections::HashSet;
 use std::fmt::{self, Formatter};
@@ -58,7 +57,7 @@ enum SourceKind {
 }
 
 /// Information to find a specific commit in a Git repository.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
 pub enum GitReference {
     /// From a tag.
     Tag(String),

src/cargo/sources/git/utils.rs

@@ -9,8 +9,7 @@ use cargo_util::{paths, ProcessBuilder};
 use curl::easy::List;
 use git2::{self, ErrorClass, ObjectType, Oid};
 use log::{debug, info};
-use serde::ser;
-use serde::Serialize;
+use serde::{ser, Deserialize, Serialize};
 use std::borrow::Cow;
 use std::env;
 use std::fmt;
@@ -79,7 +78,7 @@ impl GitRemote {
     }
 
     pub fn rev_for(&self, path: &Path, reference: &GitReference) -> CargoResult<git2::Oid> {
-        reference.resolve(&self.db_at(path)?.repo)
+        reference.resolve(&self.db_at(path)?.repo, true)
     }
 
     pub fn checkout(
@@ -104,7 +103,7 @@ impl GitRemote {
                     }
                 }
                 None => {
-                    if let Ok(rev) = reference.resolve(&db.repo) {
+                    if let Ok(rev) = reference.resolve(&db.repo, true) {
                         return Ok((db, rev));
                     }
                 }
@@ -123,7 +122,7 @@ impl GitRemote {
             .context(format!("failed to clone into: {}", into.display()))?;
         let rev = match locked_rev {
             Some(rev) => rev,
-            None => reference.resolve(&repo)?,
+            None => reference.resolve(&repo, true)?,
         };
 
         Ok((
@@ -180,12 +179,48 @@ impl GitDatabase {
     }
 
     pub fn resolve(&self, r: &GitReference) -> CargoResult<git2::Oid> {
-        r.resolve(&self.repo)
+        r.resolve(&self.repo, true)
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+struct ShalowDataBlob<'a> {
+    tree: &'a str,
+    etag: &'a str,
+}
+
+#[test]
+fn check_with_git_hub() {
+    panic!(r#"nonstandard shallow clone may be worse than a full check out. 
+    This test is here to make sure we do not merge until we have official signoff from GitHub"#)
+}
+
 impl GitReference {
-    pub fn resolve(&self, repo: &git2::Repository) -> CargoResult<git2::Oid> {
+    pub fn resolve(&self, repo: &git2::Repository, tree: bool) -> CargoResult<git2::Oid> {
+        // Check if Cargo has done a nonstandard shallow clone
+        if let Some(reference) = repo
+            .find_reference(
+                &(format!(
+                    "refs/cargo-{}",
+                    serde_json::to_string(self).expect("why cant we make json of this")
+                )),
+            )
+            .ok()
+            .and_then(|re| {
+                let blob = re.peel_to_blob().ok()?;
+                let shalow_data: ShalowDataBlob<'_> =
+                    serde_json::from_slice(blob.content()).ok()?;
+                let id = if tree {
+                    shalow_data.tree
+                } else {
+                    shalow_data.etag
+                };
+                Some(id.parse::<Oid>().ok()?)
+            })
+        {
+            return Ok(reference);
+        }
+
         let id = match self {
             // Note that we resolve the named tag here in sync with where it's
             // fetched into via `fetch` below.
@@ -707,6 +742,12 @@ fn reset(repo: &git2::Repository, obj: &git2::Object<'_>, config: &Config) -> Ca
     opts.progress(|_, cur, max| {
         drop(pb.tick(cur, max, ""));
     });
+    if obj.as_tree().is_some() {
+        debug!("doing reset for Cargo nonstandard shallow clone");
+        repo.checkout_tree(obj, Some(&mut opts))?;
+        debug!("reset done");
+        return Ok(());
+    }
     debug!("doing reset");
     repo.reset(obj, git2::ResetType::Hard, Some(&mut opts))?;
     debug!("reset done");
@@ -819,32 +860,44 @@ pub fn fetch(
     // The `+` symbol on the refspec means to allow a forced (fast-forward)
     // update which is needed if there is ever a force push that requires a
     // fast-forward.
-    match reference {
-        // For branches and tags we can fetch simply one reference and copy it
-        // locally, no need to fetch other branches/tags.
-        GitReference::Branch(b) => {
-            refspecs.push(format!("+refs/heads/{0}:refs/remotes/origin/{0}", b));
-        }
-        GitReference::Tag(t) => {
-            refspecs.push(format!("+refs/tags/{0}:refs/remotes/origin/tags/{0}", t));
+    if let Some(oid_to_fetch) = oid_to_fetch {
+        // GitHub told us exactly the min needed to fetch. So we can go ahead and do a Cargo nonstandard shallow clone.
+        refspecs.push(format!("+{0}", oid_to_fetch));
+    } else {
+        // In some cases we have Cargo nonstandard shallow cloned this repo before, but cannot do it now.
+        // Mostly if GitHub is now rate limiting us. If so, remove the info about the shallow clone.
+        if let Ok(mut refe) = repo.find_reference(&format!(
+            "refs/cargo-{}",
+            serde_json::to_string(reference).expect("why cant we make json of this")
+        )) {
+            let _ = refe.delete();
         }
 
-        GitReference::DefaultBranch => {
-            refspecs.push(String::from("+HEAD:refs/remotes/origin/HEAD"));
-        }
+        match reference {
+            // For branches and tags we can fetch simply one reference and copy it
+            // locally, no need to fetch other branches/tags.
+            GitReference::Branch(b) => {
+                refspecs.push(format!("+refs/heads/{0}:refs/remotes/origin/{0}", b));
+            }
+            GitReference::Tag(t) => {
+                refspecs.push(format!("+refs/tags/{0}:refs/remotes/origin/tags/{0}", t));
+            }
 
-        GitReference::Rev(rev) => {
-            if rev.starts_with("refs/") {
-                refspecs.push(format!("+{0}:{0}", rev));
-            } else if let Some(oid_to_fetch) = oid_to_fetch {
-                refspecs.push(format!("+{0}:refs/commit/{0}", oid_to_fetch));
-            } else {
-                // We don't know what the rev will point to. To handle this
-                // situation we fetch all branches and tags, and then we pray
-                // it's somewhere in there.
-                refspecs.push(String::from("+refs/heads/*:refs/remotes/origin/*"));
+            GitReference::DefaultBranch => {
                 refspecs.push(String::from("+HEAD:refs/remotes/origin/HEAD"));
-                tags = true;
+            }
+
+            GitReference::Rev(rev) => {
+                if rev.starts_with("refs/") {
+                    refspecs.push(format!("+{0}:{0}", rev));
+                } else {
+                    // We don't know what the rev will point to. To handle this
+                    // situation we fetch all branches and tags, and then we pray
+                    // it's somewhere in there.
+                    refspecs.push(String::from("+refs/heads/*:refs/remotes/origin/*"));
+                    refspecs.push(String::from("+HEAD:refs/remotes/origin/HEAD"));
+                    tags = true;
+                }
             }
         }
     }
@@ -1046,6 +1099,22 @@ enum FastPathRev {
     Indeterminate,
 }
 
+#[derive(Debug, Deserialize)]
+struct GithubFastPathJsonResponse {
+    sha: String,
+    commit: GithubCommitJsonResponse,
+}
+
+#[derive(Debug, Deserialize)]
+struct GithubCommitJsonResponse {
+    tree: GithubTreeJsonResponse,
+}
+
+#[derive(Debug, Deserialize)]
+struct GithubTreeJsonResponse {
+    sha: String,
+}
+
 /// Updating the index is done pretty regularly so we want it to be as fast as
 /// possible. For registries hosted on GitHub (like the crates.io index) there's
 /// a fast path available to use [1] to tell us that there's no updates to be
@@ -1067,11 +1136,8 @@ fn github_fast_path(
     config: &Config,
 ) -> CargoResult<FastPathRev> {
     let url = Url::parse(url)?;
-    if !is_github(&url) {
-        return Ok(FastPathRev::Indeterminate);
-    }
 
-    let local_object = reference.resolve(repo).ok();
+    let local_object = reference.resolve(repo, false).ok();
 
     let github_branch_name = match reference {
         GitReference::Branch(branch) => branch,
@@ -1111,6 +1177,10 @@ fn github_fast_path(
         }
     };
 
+    if !is_github(&url) {
+        return Ok(FastPathRev::Indeterminate);
+    }
+
     // This expects GitHub urls in the form `github.com/user/repo` and nothing
     // else
     let mut pieces = url
@@ -1141,7 +1211,7 @@ fn github_fast_path(
     handle.useragent("cargo")?;
     handle.http_headers({
         let mut headers = List::new();
-        headers.append("Accept: application/vnd.github.3.sha")?;
+        headers.append("Accept: application/vnd.github+json")?;
         if let Some(local_object) = local_object {
             headers.append(&format!("If-None-Match: \"{}\"", local_object))?;
         }
@@ -1161,7 +1231,24 @@ fn github_fast_path(
     if response_code == 304 {
         Ok(FastPathRev::UpToDate)
     } else if response_code == 200 {
-        let oid_to_fetch = str::from_utf8(&response_body)?.parse::<Oid>()?;
+        let data: GithubFastPathJsonResponse = serde_json::from_slice(&response_body)?;
+        // We can do a Cargo nonstandard shallow clone, so record the relevant information.
+        let bytes = serde_json::to_string(&ShalowDataBlob {
+            tree: &data.commit.tree.sha,
+            etag: &data.sha,
+        })
+        .expect("why cant we make json of this");
+        let shallow_blob = repo.blob(bytes.as_bytes())?;
+        repo.reference(
+            &format!(
+                "refs/cargo-{}",
+                serde_json::to_string(reference).expect("why cant we make json of this")
+            ),
+            shallow_blob,
+            true,
+            "",
+        )?;
+        let oid_to_fetch = str::from_utf8(data.commit.tree.sha.as_bytes())?.parse::<Oid>()?;
         Ok(FastPathRev::NeedsFetch(oid_to_fetch))
     } else {
         // Usually response_code == 404 if the repository does not exist, and

src/cargo/sources/registry/remote.rs

@@ -102,7 +102,7 @@ impl<'cfg> RemoteRegistry<'cfg> {
             }
         }
         let repo = self.repo()?;
-        let oid = self.index_git_ref.resolve(repo)?;
+        let oid = self.index_git_ref.resolve(repo, true)?;
         let obj = repo.find_object(oid, None)?;
         let tree = obj.peel_to_tree()?;