Merge remote-tracking branch 'origin/master' into large-path-warning

doc/manual/rl-next/shallow-git-fetching-by-default.md

@@ -0,0 +1,12 @@
+---
+synopsis: "`fetchTree` now fetches git repositories shallowly by default"
+prs: 10028
+---
+
+`builtins.fetchTree` now clones git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases.
+
+Previously, the default behavior was to clone the full history of a specific tag or branch (eg. `ref`) and only afterwards extract the files of one specific revision.
+
+From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`.
+
+The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`.

maintainers/upload-release.pl

@@ -243,6 +243,7 @@ write_file("$tmpDir/fallback-paths.nix",
     "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
     "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
     "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+    "  riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
     "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
     "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
     "}\n");

scripts/install.in

@@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in
         path=@tarballPath_armv7l-linux@
         system=armv7l-linux
         ;;
+    Linux.riscv64)
+        hash=@tarballHash_riscv64-linux@
+        path=@tarballPath_riscv64-linux@
+        system=riscv64-linux
+        ;;
     Darwin.x86_64)
         hash=@tarballHash_x86_64-darwin@
         path=@tarballPath_x86_64-darwin@

src/libexpr/primops/fetchTree.cc

@@ -137,6 +137,11 @@ static void fetchTree(
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
+        // fetchTree should fetch git repos with shallow = true by default
+        if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
+            attrs.emplace("shallow", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>(
@@ -320,6 +325,8 @@ static RegisterPrimOp primop_fetchTree({
 
         - `ref` (String, optional)
 
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
           A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
 
           Default: `"HEAD"`
@@ -333,8 +340,9 @@ static RegisterPrimOp primop_fetchTree({
         - `shallow` (Bool, optional)
 
           Make a shallow clone when fetching the Git tree.
+          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
 
-          Default: `false`
+          Default: `true`
 
         - `submodules` (Bool, optional)
 
@@ -344,8 +352,11 @@ static RegisterPrimOp primop_fetchTree({
 
         - `allRefs` (Bool, optional)
 
-          If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
-          Otherwise, only the latest commit is fetched if it is not already cached.
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          Whether to fetch all references (eg. branches and tags) of the repository.
+          With this argument being true, it's possible to load a `rev` from *any* `ref`.
+          (Without setting this option, only `rev`s from the specified `ref` are supported).
 
           Default: `false`
 
@@ -599,6 +610,8 @@ static RegisterPrimOp primop_fetchGit({
 
         [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
 
+        This option has no effect once `shallow` cloning is enabled.
+
         By default, the `ref` value is prefixed with `refs/heads/`.
         As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`.
 
@@ -616,13 +629,15 @@ static RegisterPrimOp primop_fetchGit({
       - `shallow` (default: `false`)
 
         Make a shallow clone when fetching the Git tree.
-
+        When this is enabled, the options `ref` and `allRefs` have no effect anymore.
       - `allRefs`
 
-        Whether to fetch all references of the repository.
-        With this argument being true, it's possible to load a `rev` from *any* `ref`
+        Whether to fetch all references (eg. branches and tags) of the repository.
+        With this argument being true, it's possible to load a `rev` from *any* `ref`.
         (by default only `rev`s from the specified `ref` are supported).
 
+        This option has no effect once `shallow` cloning is enabled.
+
       - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
 
         Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.

src/libutil/hash.cc

@@ -50,21 +50,14 @@ bool Hash::operator == (const Hash & h2) const
 }
 
 
-bool Hash::operator != (const Hash & h2) const
+std::strong_ordering Hash::operator <=> (const Hash & h) const
 {
-    return !(*this == h2);
-}
-
-
-bool Hash::operator < (const Hash & h) const
-{
-    if (hashSize < h.hashSize) return true;
-    if (hashSize > h.hashSize) return false;
+    if (auto cmp = algo <=> h.algo; cmp != 0) return cmp;
+    if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp;
     for (unsigned int i = 0; i < hashSize; i++) {
-        if (hash[i] < h.hash[i]) return true;
-        if (hash[i] > h.hash[i]) return false;
+        if (auto cmp = hash[i] <=> h.hash[i]; cmp != 0) return cmp;
     }
-    return false;
+    return std::strong_ordering::equivalent;
 }
 
 

src/libutil/hash.hh

@@ -86,19 +86,14 @@ private:
 
 public:
     /**
-     * Check whether two hash are equal.
+     * Check whether two hashes are equal.
      */
     bool operator == (const Hash & h2) const;
 
     /**
-     * Check whether two hash are not equal.
+     * Compare how two hashes are ordered.
      */
-    bool operator != (const Hash & h2) const;
-
-    /**
-     * For sorting.
-     */
-    bool operator < (const Hash & h) const;
+    std::strong_ordering operator <=> (const Hash & h2) const;
 
     /**
      * Returns the length of a base-16 representation of this hash.

src/libutil/url.cc

@@ -171,16 +171,16 @@ std::string fixGitURL(const std::string & url)
     std::regex scpRegex("([^/]*)@(.*):(.*)");
     if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex))
         return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3");
-    else {
-        if (url.find("://") == std::string::npos) {
-            return (ParsedURL {
-                .scheme = "file",
-                .authority = "",
-                .path = url
-            }).to_string();
-        } else
-            return url;
+    if (hasPrefix(url, "file:"))
+        return url;
+    if (url.find("://") == std::string::npos) {
+        return (ParsedURL {
+            .scheme = "file",
+            .authority = "",
+            .path = url
+        }).to_string();
     }
+    return url;
 }
 
 // https://www.rfc-editor.org/rfc/rfc3986#section-3.1

tests/functional/flakes/eval-cache.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 source ./common.sh
 
 requireGit

tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix

@@ -0,0 +1,45 @@
+{
+  description = "fetchTree fetches git repos shallowly by default";
+  script = ''
+    # purge nix git cache to make sure we start with a clean slate
+    client.succeed("rm -rf ~/.cache/nix")
+
+    # add two commits to the repo:
+    #   - one with a large file (2M)
+    #   - another one making the file small again
+    client.succeed(f"""
+      dd if=/dev/urandom of={repo.path}/thailand bs=1M count=2 \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1' \
+      && echo 'ThaigerSprint' > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit2' \
+      && {repo.git} push origin main
+    """)
+
+    # memoize the revision
+    commit2_rev = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # construct the fetcher call
+    fetchGit_expr = f"""
+      builtins.fetchTree {{
+        type = "git";
+        url = "{repo.remote}";
+        rev = "{commit2_rev}";
+      }}
+    """
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr '({fetchGit_expr}).outPath'
+    """)
+
+    # check that the size of ~/.cache/nix is less than 1M
+    cache_size = client.succeed("""
+      du -s ~/.cache/nix
+    """).strip().split()[0]
+    assert int(cache_size) < 1024, f"cache size is {cache_size}K which is larger than 1M"
+  '';
+}