From 15833516a4bad0a4ae7786293b22df4bf650aa80 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 5 Feb 2019 16:42:45 -0500
Subject: [PATCH 01/90] Add armv6l-linux & armv7l-linux as cross jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is a cheap way to get 32-bit ARM working. We don’t support it
officially but lots of people have raspberry pis and similar hardware
they want to install the Nix package manager on.
---
 release.nix | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/release.nix b/release.nix
index 271645067..0ec742906 100644
--- a/release.nix
+++ b/release.nix
@@ -2,6 +2,7 @@
 , nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+, crossSystems ? [ "armv6l-linux" "armv7l-linux" ]
 }:
 
 let
@@ -53,11 +54,12 @@ let
       };
 
 
-    build = pkgs.lib.genAttrs systems (system:
+    build = pkgs.lib.genAttrs (systems ++ crossSystems) (system:
 
-      let pkgs = import nixpkgs { inherit system; }; in
-
-      with pkgs;
+      let pkgs = if builtins.elem system systems
+                 then import nixpkgs { inherit system; }
+                 else import nixpkgs { crossSystem = { inherit system; }; };
+      in with pkgs;
 
       with import ./release-common.nix { inherit pkgs; };
 
@@ -89,9 +91,12 @@ let
       });
 
 
-    perlBindings = pkgs.lib.genAttrs systems (system:
+    perlBindings = pkgs.lib.genAttrs (systems ++ crossSystems) (system:
 
-      let pkgs = import nixpkgs { inherit system; }; in with pkgs;
+      let pkgs = if builtins.elem system systems
+                 then import nixpkgs { inherit system; }
+                 else import nixpkgs { crossSystem = { inherit system; }; };
+      in with pkgs;
 
       releaseTools.nixBuild {
         name = "nix-perl";
@@ -112,9 +117,12 @@ let
       });
 
 
-    binaryTarball = pkgs.lib.genAttrs systems (system:
+    binaryTarball = pkgs.lib.genAttrs (systems ++ crossSystems) (system:
 
-      with import nixpkgs { inherit system; };
+      let pkgs = if builtins.elem system systems
+                 then import nixpkgs { inherit system; }
+                 else import nixpkgs { crossSystem = { inherit system; }; };
+      in with pkgs;
 
       let
         toplevel = builtins.getAttr system jobs.build;

From 1996af425ac8ddea1e8a591650e7d0caba2aa201 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 6 Feb 2019 21:43:47 -0500
Subject: [PATCH 02/90] Use buildPackages for native dependencies

Unfortunately, releaseTools.nixBuild does not separate native and
non-native build inputs. As an alternative, we can just use
buildPackages to get the native version of some packages like:

- pkgconfig
- git
- curl
- utillinux
---
 release-common.nix | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index 4c5565985..f2aa57c2c 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -50,14 +50,16 @@ rec {
   buildDeps =
     [ curl
       bzip2 xz brotli editline
-      openssl pkgconfig sqlite boehmgc
+      openssl sqlite boehmgc
       boost
 
+      buildPackages.pkgconfig
+
       # Tests
-      git
-      mercurial
+      buildPackages.git
+      buildPackages.mercurial
     ]
-    ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
+    ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       ((aws-sdk-cpp.override {

From e9072ded9749ab00cc397980e8a26f83d341efc0 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 6 Feb 2019 22:43:28 -0500
Subject: [PATCH 03/90] Use nativeBuildInputs

---
 release-common.nix | 16 +++++++++-------
 release.nix        |  3 +++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index f2aa57c2c..707d36f95 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -47,19 +47,21 @@ rec {
       autoreconfHook
     ];
 
+  nativeBuildDeps =
+    [ buildPackages.pkgconfig
+
+      # Tests
+      buildPackages.git
+      buildPackages.mercurial
+    ] ++ lib.optional stdenv.isLinux buildPackages.utillinuxMinimal;
+
   buildDeps =
     [ curl
       bzip2 xz brotli editline
       openssl sqlite boehmgc
       boost
-
-      buildPackages.pkgconfig
-
-      # Tests
-      buildPackages.git
-      buildPackages.mercurial
     ]
-    ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal]
+    ++ lib.optional stdenv.isLinux libseccomp
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       ((aws-sdk-cpp.override {
diff --git a/release.nix b/release.nix
index 0ec742906..9843efa29 100644
--- a/release.nix
+++ b/release.nix
@@ -24,6 +24,7 @@ let
         src = nix;
         inherit officialRelease;
 
+        nativeBuildInputs = nativeBuildDeps;
         buildInputs = tarballDeps ++ buildDeps;
 
         configureFlags = "--enable-gc";
@@ -67,6 +68,7 @@ let
         name = "nix";
         src = tarball;
 
+        nativeBuildInputs = nativeBuildDeps;
         buildInputs = buildDeps;
 
         preConfigure =
@@ -199,6 +201,7 @@ let
         name = "nix-build";
         src = tarball;
 
+        nativeBuildInputs = nativeBuildDeps;
         buildInputs = buildDeps;
 
         dontInstall = false;

From f6ea56dfac1f4df45a5fa9e2801bc632dee9eff7 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 6 Feb 2019 23:04:40 -0500
Subject: [PATCH 04/90] Get shellcheck from buildPackages

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 9843efa29..06db7bdc0 100644
--- a/release.nix
+++ b/release.nix
@@ -133,7 +133,7 @@ let
       in
 
       runCommand "nix-binary-tarball-${version}"
-        { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+        { nativeBuildInputs = lib.optional (system != "aarch64-linux") buildPackages.shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''

From 4fefe26717fa70828e3f524e43c76e3f7b7a09b0 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 5 Feb 2021 18:22:34 -0600
Subject: [PATCH 05/90] Re-enable armv6l support

This fixes the libatomic detection.
---
 configure.ac | 2 +-
 flake.nix    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 2047ed8d2..685c471c5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -152,7 +152,7 @@ int main() {
 }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
 AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
 if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
-    LIBS="-latomic $LIBS"
+    LDFLAGS="$LDFLAGS -latomic"
 fi
 
 PKG_PROG_PKG_CONFIG
diff --git a/flake.nix b/flake.nix
index 869b92cb7..7e02fd70d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -20,7 +20,7 @@
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
       systems = linuxSystems ++ [ "x86_64-darwin" ];
 
-      crossSystems = [ "armv7l-linux" ];
+      crossSystems = [ "armv6l-linux" "armv7l-linux" ];
 
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
 

From 5f07f2ff2b964bec71be34cf6cf1265ce88b7490 Mon Sep 17 00:00:00 2001
From: Alyssa Ross <hi@alyssa.is>
Date: Wed, 5 May 2021 13:56:23 +0000
Subject: [PATCH 06/90] doc: builtins: use a definition list

This looks a lot better (and is also more semantically meaningful).

Since this list is generated in a Nix expression, I don't think using
HTML here is going to be the thing that puts people off modifying this
part of the documentation!
---
 doc/manual/generate-builtins.nix              | 10 ++++++----
 doc/manual/local.mk                           |  1 +
 doc/manual/src/expressions/builtins-prefix.md |  9 +++++----
 doc/manual/src/expressions/builtins-suffix.md |  1 +
 4 files changed, 13 insertions(+), 8 deletions(-)
 create mode 100644 doc/manual/src/expressions/builtins-suffix.md

diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
index 416a7fdba..92c7b1a31 100644
--- a/doc/manual/generate-builtins.nix
+++ b/doc/manual/generate-builtins.nix
@@ -6,9 +6,11 @@ builtins:
 concatStrings (map
   (name:
     let builtin = builtins.${name}; in
-    "  - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args)
-    + "  \n\n"
-    + concatStrings (map (s: "    ${s}\n") (splitLines builtin.doc)) + "\n\n"
+    "<dt><code>${name} "
+    + concatStringsSep " " (map (s: "<var>${s}</var>") builtin.args)
+    + "</code></dt>"
+    + "<dd>\n\n"
+    + builtin.doc
+    + "\n\n</dd>"
   )
   (attrNames builtins))
-
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 271529b38..e25157af8 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -64,6 +64,7 @@ $(d)/conf-file.json: $(bindir)/nix
 $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	@cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/builtins.json: $(bindir)/nix
diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md
index c16b2805f..87127de2a 100644
--- a/doc/manual/src/expressions/builtins-prefix.md
+++ b/doc/manual/src/expressions/builtins-prefix.md
@@ -9,7 +9,8 @@ scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `derivation` *attrs*; `builtins.derivation` *attrs*\
-
-    `derivation` is described in [its own section](derivations.md).
-
+<dl>
+  <dt><code>derivation <var>attrs</var></code>;
+      <code>builtins.derivation <var>attrs</var></code></dt>
+  <dd><p><var>derivation</var> in described in
+         <a href="derivations.md">its own section</a>.</p></dd>
diff --git a/doc/manual/src/expressions/builtins-suffix.md b/doc/manual/src/expressions/builtins-suffix.md
new file mode 100644
index 000000000..a74db2857
--- /dev/null
+++ b/doc/manual/src/expressions/builtins-suffix.md
@@ -0,0 +1 @@
+</dl>

From 7616268812cc2fe141132749d6d667b5c4f4d877 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 7 May 2021 11:06:17 +0200
Subject: [PATCH 07/90] Always send the realisations as JSON
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Align all the worker protocol with `buildDerivation` which inlines the
realisations as one opaque json blob.
That way we don’t have to bother changing the remote store protocol
when the definition of `Realisation` changes, as long as we keep the
json backwards-compatible
---
 src/libstore/daemon.cc          | 25 ++++++++++++++++++-------
 src/libstore/remote-store.cc    | 23 +++++++++++++++++------
 src/libstore/worker-protocol.hh |  2 +-
 3 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 72b3e3e13..e06fb9ce2 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -885,10 +885,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopRegisterDrvOutput: {
         logger->startWork();
-        auto outputId = DrvOutput::parse(readString(from));
-        auto outputPath = StorePath(readString(from));
-        store->registerDrvOutput(Realisation{
-            .id = outputId, .outPath = outputPath});
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            auto outputId = DrvOutput::parse(readString(from));
+            auto outputPath = StorePath(readString(from));
+            store->registerDrvOutput(Realisation{
+                .id = outputId, .outPath = outputPath});
+        } else {
+            auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
+            store->registerDrvOutput(realisation);
+        }
         logger->stopWork();
         break;
     }
@@ -898,9 +903,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto outputId = DrvOutput::parse(readString(from));
         auto info = store->queryRealisation(outputId);
         logger->stopWork();
-        std::set<StorePath> outPaths;
-        if (info) outPaths.insert(info->outPath);
-        worker_proto::write(*store, to, outPaths);
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            std::set<StorePath> outPaths;
+            if (info) outPaths.insert(info->outPath);
+            worker_proto::write(*store, to, outPaths);
+        } else {
+            std::set<Realisation> realisations;
+            if (info) realisations.insert(*info);
+            worker_proto::write(*store, to, realisations);
+        }
         break;
     }
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index d9b6e9488..aec243637 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -653,8 +653,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
 {
     auto conn(getConnection());
     conn->to << wopRegisterDrvOutput;
-    conn->to << info.id.to_string();
-    conn->to << std::string(info.outPath.to_string());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+        conn->to << info.id.to_string();
+        conn->to << std::string(info.outPath.to_string());
+    } else {
+        worker_proto::write(*this, conn->to, info);
+    }
     conn.processStderr();
 }
 
@@ -664,10 +668,17 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
     conn->to << wopQueryRealisation;
     conn->to << id.to_string();
     conn.processStderr();
-    auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
-    if (outPaths.empty())
-        return std::nullopt;
-    return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+        auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
+        if (outPaths.empty())
+            return std::nullopt;
+        return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+    } else {
+        auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{});
+        if (realisations.empty())
+            return std::nullopt;
+        return *realisations.begin();
+    }
 }
 
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index fdd692cf0..e89183d40 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 30)
+#define PROTOCOL_VERSION (1 << 8 | 31)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

From 7ce0441d806d12bb073047337545e3901404996d Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 10 Nov 2020 14:49:25 +0100
Subject: [PATCH 08/90] Add a dependencies field to DrvOutputInfo

Currently never used, nor set but will be useful shortly
---
 src/libstore/realisation.cc | 10 ++++++++++
 src/libstore/realisation.hh |  2 ++
 2 files changed, 12 insertions(+)

diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index 638065547..27ad6c150 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -22,10 +22,14 @@ std::string DrvOutput::to_string() const {
 }
 
 nlohmann::json Realisation::toJSON() const {
+    nlohmann::json jsonDrvOutputDeps;
+    for (auto & dep : drvOutputDeps)
+        jsonDrvOutputDeps.push_back(dep.to_string());
     return nlohmann::json{
         {"id", id.to_string()},
         {"outPath", outPath.to_string()},
         {"signatures", signatures},
+        {"drvOutputDeps", jsonDrvOutputDeps},
     };
 }
 
@@ -51,10 +55,16 @@ Realisation Realisation::fromJSON(
     if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
         signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
 
+    std::set <DrvOutput> drvOutputDeps;
+    if (auto jsonDependencies = json.find("drvOutputDeps"); jsonDependencies != json.end())
+        for (auto & jsonDep : *jsonDependencies)
+            drvOutputDeps.insert(DrvOutput::parse(jsonDep.get<std::string>()));
+
     return Realisation{
         .id = DrvOutput::parse(getField("id")),
         .outPath = StorePath(getField("outPath")),
         .signatures = signatures,
+        .drvOutputDeps = drvOutputDeps,
     };
 }
 
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index f5049c9e9..1e2808ce3 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -28,6 +28,8 @@ struct Realisation {
 
     StringSet signatures;
 
+    std::set<DrvOutput> drvOutputDeps;
+
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 

From eca6ff06d611ef005e80d419e1b6050393fc056d Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 7 May 2021 13:57:01 +0200
Subject: [PATCH 09/90] Store the realisation deps on the local store

---
 src/libstore/ca-specific-schema.sql | 11 +++++-
 src/libstore/local-store.cc         | 61 +++++++++++++++++++++++------
 2 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql
index 20ee046a1..08af0cc1f 100644
--- a/src/libstore/ca-specific-schema.sql
+++ b/src/libstore/ca-specific-schema.sql
@@ -3,10 +3,19 @@
 -- is enabled
 
 create table if not exists Realisations (
+    id integer primary key autoincrement not null,
     drvPath text not null,
     outputName text not null, -- symbolic output id, usually "out"
     outputPath integer not null,
     signatures text, -- space-separated list
-    primary key (drvPath, outputName),
     foreign key (outputPath) references ValidPaths(id) on delete cascade
 );
+
+create index if not exists IndexRealisations on Realisations(drvPath, outputName);
+
+create table if not exists RealisationsRefs (
+    referrer integer not null,
+    realisationReference integer,
+    foreign key (referrer) references Realisations(id) on delete cascade,
+    foreign key (realisationReference) references Realisations(id) on delete restrict
+);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 83daa7506..f8d55621d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -59,6 +59,8 @@ struct LocalStore::State::Stmts {
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
+    SQLiteStmt QueryRealisationRealisationReferences;
+    SQLiteStmt AddRealisationRealisationReference;
 };
 
 int getSchema(Path schemaPath)
@@ -316,7 +318,7 @@ LocalStore::LocalStore(const Params & params)
             )");
         state->stmts->QueryRealisedOutput.create(state->db,
             R"(
-                select Output.path, Realisations.signatures from Realisations
+                select Realisations.id, Output.path, Realisations.signatures from Realisations
                     inner join ValidPaths as Output on Output.id = Realisations.outputPath
                     where drvPath = ? and outputName = ?
                     ;
@@ -328,6 +330,19 @@ LocalStore::LocalStore(const Params & params)
                     where drvPath = ?
                     ;
             )");
+        state->stmts->QueryRealisationRealisationReferences.create(state->db,
+            R"(
+                select drvPath, outputName from Realisations
+                    join RealisationsRefs on realisationReference = Realisations.id
+                    where referrer = ?;
+            )");
+        state->stmts->AddRealisationRealisationReference.create(state->db,
+            R"(
+                insert or replace into RealisationsRefs (referrer, realisationReference)
+                values (
+                    ?,
+                    (select id from Realisations where drvPath = ? and outputName = ?));
+            )");
     }
 }
 
@@ -666,13 +681,17 @@ void LocalStore::registerDrvOutput(const Realisation & info)
     settings.requireExperimentalFeature("ca-derivations");
     auto state(_state.lock());
     retrySQLite<void>([&]() {
-        state->stmts->RegisterRealisedOutput.use()
-            (info.id.strHash())
-            (info.id.outputName)
-            (printStorePath(info.outPath))
-            (concatStringsSep(" ", info.signatures))
+        state->stmts->RegisterRealisedOutput
+            .use()(info.id.strHash())(info.id.outputName)(printStorePath(
+                info.outPath))(concatStringsSep(" ", info.signatures))
             .exec();
     });
+    uint64_t myId = state->db.getLastInsertedRowId();
+    for (auto& outputId : info.drvOutputDeps) {
+        state->stmts->AddRealisationRealisationReference
+            .use()(myId)(outputId.strHash())(outputId.outputName)
+            .exec();
+    }
 }
 
 void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
@@ -1670,14 +1689,32 @@ std::optional<const Realisation> LocalStore::queryRealisation(
     typedef std::optional<const Realisation> Ret;
     return retrySQLite<Ret>([&]() -> Ret {
         auto state(_state.lock());
-        auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
-            id.outputName));
-        if (!use.next())
+        auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()(
+            id.strHash())(id.outputName));
+        if (!useQueryRealisedOutput.next())
             return std::nullopt;
-        auto outputPath = parseStorePath(use.getStr(0));
-        auto signatures = tokenizeString<StringSet>(use.getStr(1));
+        auto realisationDbId = useQueryRealisedOutput.getInt(0);
+        auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
+        auto signatures =
+            tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
+
+        std::set<DrvOutput> drvOutputDeps;
+        auto useRealisationRefs(
+            state->stmts->QueryRealisationRealisationReferences.use()(
+                realisationDbId));
+        while (useRealisationRefs.next())
+            drvOutputDeps.insert(DrvOutput{
+                    Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
+                useRealisationRefs.getStr(1),
+            }
+        );
+
         return Ret{Realisation{
-            .id = id, .outPath = outputPath, .signatures = signatures}};
+            .id = id,
+            .outPath = outputPath,
+            .signatures = signatures,
+            .drvOutputDeps = drvOutputDeps,
+        }};
     });
 }
 }  // namespace nix

From af3afd25eafb7866406aee04ef049121a3e3ffb0 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 10:26:58 +0200
Subject: [PATCH 10/90] Add a method to compute the closure of a realisation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Only considers the closure in term of `Realisation`, ignores all the
opaque inputs.

Dunno whether that’s the nicest solution, need to think it through a bit
---
 src/libstore/realisation.cc | 38 +++++++++++++++++++++++++++++++++++++
 src/libstore/realisation.hh |  3 +++
 2 files changed, 41 insertions(+)

diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index 27ad6c150..fab10f68c 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -1,5 +1,6 @@
 #include "realisation.hh"
 #include "store-api.hh"
+#include "closure.hh"
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -21,6 +22,43 @@ std::string DrvOutput::to_string() const {
     return strHash() + "!" + outputName;
 }
 
+std::set<Realisation> Realisation::closure(Store & store, std::set<Realisation> startOutputs)
+{
+    std::set<Realisation> res;
+    Realisation::closure(store, startOutputs, res);
+    return res;
+}
+
+void Realisation::closure(Store & store, std::set<Realisation> startOutputs, std::set<Realisation> & res)
+{
+    auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
+        std::set<Realisation> res;
+        for (auto& currentDep : current.drvOutputDeps) {
+            if (auto currentRealisation = store.queryRealisation(currentDep))
+                res.insert(*currentRealisation);
+            else
+                throw Error(
+                    "Unrealised derivation '%s'", currentDep.to_string());
+        }
+        return res;
+    };
+
+    computeClosure<Realisation>(
+        startOutputs, res,
+        [&](const Realisation& current,
+            std::function<void(std::promise<std::set<Realisation>>&)>
+                processEdges) {
+            std::promise<std::set<Realisation>> promise;
+            try {
+                auto res = getDeps(current);
+                promise.set_value(res);
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+            return processEdges(promise);
+        });
+}
+
 nlohmann::json Realisation::toJSON() const {
     nlohmann::json jsonDrvOutputDeps;
     for (auto & dep : drvOutputDeps)
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 1e2808ce3..776ab606c 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -38,6 +38,9 @@ struct Realisation {
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 
+    static std::set<Realisation> closure(Store &, std::set<Realisation>);
+    static void closure(Store &, std::set<Realisation>, std::set<Realisation>& res);
+
     StorePath getPath() const { return outPath; }
 
     GENERATE_CMP(Realisation, me->id, me->outPath);

From 8c30acc3e896839283a2c96ec97cc0c811e8ad6a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 10:35:31 +0200
Subject: [PATCH 11/90] Properly track the drvoutput references when building

---
 src/libstore/build/derivation-goal.cc |  1 +
 src/libstore/misc.cc                  | 42 +++++++++++++++++++++++++++
 src/libstore/store-api.hh             |  5 ++++
 3 files changed, 48 insertions(+)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 9100d3333..93fc54440 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -927,6 +927,7 @@ void DerivationGoal::resolvedFinished() {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             newRealisation.signatures.clear();
+            newRealisation.drvOutputDeps = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
         } else {
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index bc5fd968c..8793f9f4a 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -254,5 +254,47 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         }});
 }
 
+std::set<DrvOutput> drvOutputReferences(
+    const std::set<Realisation> inputRealisations,
+    const StorePathSet pathReferences)
+{
+    std::set<DrvOutput> res;
 
+    std::map<StorePath, std::set<DrvOutput>> inputsByOutputPath;
+    for (const auto & input : inputRealisations)
+        inputsByOutputPath[input.outPath].insert(input.id);
+
+    for (const auto & path : pathReferences) {
+        auto theseInputs = inputsByOutputPath[path];
+        res.insert(theseInputs.begin(), theseInputs.end());
+    }
+
+    return res;
+}
+
+std::set<DrvOutput> drvOutputReferences(
+    Store & store,
+    const Derivation & drv,
+    const StorePath & outputPath)
+{
+    std::set<Realisation> inputRealisations;
+
+    for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
+        auto outputHashes =
+            staticOutputHashes(store, store.readDerivation(inputDrv));
+        for (const auto& outputName : outputNames) {
+            auto thisRealisation = store.queryRealisation(
+                DrvOutput{outputHashes.at(outputName), outputName});
+            if (!thisRealisation)
+                throw Error(
+                    "output '%s' of derivation '%s' isn’t built", outputName,
+                    store.printStorePath(inputDrv));
+            inputRealisations.insert(*thisRealisation);
+        }
+    }
+
+    auto info = store.queryPathInfo(outputPath);
+
+    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
+}
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f66298991..29af0f495 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -864,4 +864,9 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri)
 
 std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
 
+std::set<DrvOutput> drvOutputReferences(
+    Store & store,
+    const Derivation & drv,
+    const StorePath & outputPath);
+
 }

From 63ebfc73c56978665bf6c58c3e4ad84485355b1a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 10:36:48 +0200
Subject: [PATCH 12/90] Make `copyPaths` copy the whole realisations closure

Otherwise registering the realisations on the remote side might fail as
it now expects a complete closure
---
 src/libstore/store-api.cc | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 93fcb068f..08573f709 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -780,20 +780,40 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
     StorePathSet storePaths;
-    std::set<Realisation> realisations;
+    std::set<Realisation> toplevelRealisations;
     for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto realisation = std::get_if<Realisation>(&path.raw)) {
             settings.requireExperimentalFeature("ca-derivations");
-            realisations.insert(*realisation);
+            toplevelRealisations.insert(*realisation);
         }
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
+
+    ThreadPool pool;
+
     try {
-        for (auto & realisation : realisations) {
-            dstStore->registerDrvOutput(realisation, checkSigs);
-        }
-    } catch (MissingExperimentalFeature & e) {
+        // Copy the realisation closure
+        processGraph<Realisation>(
+            pool, Realisation::closure(*srcStore, toplevelRealisations),
+            [&](const Realisation& current) -> std::set<Realisation> {
+                std::set<Realisation> children;
+                for (const auto& drvOutput : current.drvOutputDeps) {
+                    auto currentChild = srcStore->queryRealisation(drvOutput);
+                    if (!currentChild)
+                        throw Error(
+                            "Incomplete realisation closure: '%s' is a "
+                            "dependency "
+                            "of '%s' but isn’t registered",
+                            drvOutput.to_string(), current.id.to_string());
+                    children.insert(*currentChild);
+                }
+                return children;
+            },
+            [&](const Realisation& current) -> void {
+                dstStore->registerDrvOutput(current, checkSigs);
+            });
+    } catch (MissingExperimentalFeature& e) {
         // Don't fail if the remote doesn't support CA derivations is it might
         // not be within our control to change that, and we might still want
         // to at least copy the output paths.

From cb46d70794b8677b7927e6ae3d492e6db886c7aa Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 12:30:12 +0200
Subject: [PATCH 13/90] Add a db migration script

---
 src/libstore/local-store.cc | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index f8d55621d..ce0e94865 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -78,7 +78,7 @@ int getSchema(Path schemaPath)
 
 void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 {
-    const int nixCASchemaVersion = 1;
+    const int nixCASchemaVersion = 2;
     int curCASchema = getSchema(schemaPath);
     if (curCASchema != nixCASchemaVersion) {
         if (curCASchema > nixCASchemaVersion) {
@@ -96,7 +96,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
               #include "ca-specific-schema.sql.gen.hh"
                 ;
             db.exec(schema);
+            curCASchema = nixCASchemaVersion;
         }
+
+        if (curCASchema < 2) {
+            SQLiteTxn txn(db);
+            // Ugly little sql dance to add a new `id` column and make it the primary key
+            db.exec(R"(
+                create table Realisations2 (
+                    id integer primary key autoincrement not null,
+                    drvPath text not null,
+                    outputName text not null, -- symbolic output id, usually "out"
+                    outputPath integer not null,
+                    signatures text, -- space-separated list
+                    foreign key (outputPath) references ValidPaths(id) on delete cascade
+                );
+                insert into Realisations2 (drvPath, outputName, outputPath, signatures)
+                    select drvPath, outputName, outputPath, signatures from Realisations;
+                drop table Realisations;
+                alter table Realisations2 rename to Realisations;
+            )");
+            db.exec(R"(
+                create index if not exists IndexRealisations on Realisations(drvPath, outputName);
+
+                create table if not exists RealisationsRefs (
+                    referrer integer not null,
+                    realisationReference integer,
+                    foreign key (referrer) references Realisations(id) on delete cascade,
+                    foreign key (realisationReference) references Realisations(id) on delete restrict
+                );
+            )");
+            txn.commit();
+        }
+
         writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
         lockFile(lockFd.get(), ltRead, true);
     }

From 1f3ff0d193c270f7b97af4aa3e463be01dbe5f2d Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 26 May 2021 16:09:02 +0200
Subject: [PATCH 14/90] Aso track the output path of the realisation
 dependencies

---
 src/libstore/build/derivation-goal.cc |  2 +-
 src/libstore/local-store.cc           | 34 +++++++++++++++------------
 src/libstore/misc.cc                  | 17 ++++++--------
 src/libstore/realisation.cc           | 20 ++++++++--------
 src/libstore/realisation.hh           |  8 ++++++-
 src/libstore/store-api.cc             |  2 +-
 src/libstore/store-api.hh             |  2 +-
 7 files changed, 46 insertions(+), 39 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 93fc54440..8c9ef0101 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -927,7 +927,7 @@ void DerivationGoal::resolvedFinished() {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             newRealisation.signatures.clear();
-            newRealisation.drvOutputDeps = drvOutputReferences(worker.store, *drv, realisation->outPath);
+            newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
         } else {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ce0e94865..debe70037 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -711,19 +711,19 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
     settings.requireExperimentalFeature("ca-derivations");
-    auto state(_state.lock());
     retrySQLite<void>([&]() {
+        auto state(_state.lock());
         state->stmts->RegisterRealisedOutput
             .use()(info.id.strHash())(info.id.outputName)(printStorePath(
                 info.outPath))(concatStringsSep(" ", info.signatures))
             .exec();
+        uint64_t myId = state->db.getLastInsertedRowId();
+        for (auto & [outputId, _] : info.dependentRealisations) {
+            state->stmts->AddRealisationRealisationReference
+                .use()(myId)(outputId.strHash())(outputId.outputName)
+                .exec();
+        }
     });
-    uint64_t myId = state->db.getLastInsertedRowId();
-    for (auto& outputId : info.drvOutputDeps) {
-        state->stmts->AddRealisationRealisationReference
-            .use()(myId)(outputId.strHash())(outputId.outputName)
-            .exec();
-    }
 }
 
 void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
@@ -1730,22 +1730,26 @@ std::optional<const Realisation> LocalStore::queryRealisation(
         auto signatures =
             tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
 
-        std::set<DrvOutput> drvOutputDeps;
+        std::map<DrvOutput, StorePath> dependentRealisations;
         auto useRealisationRefs(
             state->stmts->QueryRealisationRealisationReferences.use()(
                 realisationDbId));
-        while (useRealisationRefs.next())
-            drvOutputDeps.insert(DrvOutput{
-                    Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
-                useRealisationRefs.getStr(1),
-            }
-        );
+        while (useRealisationRefs.next()) {
+            auto depHash = useRealisationRefs.getStr(0);
+            auto depOutputName = useRealisationRefs.getStr(1);
+            auto useQueryRealisedOutput(
+                state->stmts->QueryRealisedOutput.use()(depHash)(depOutputName));
+            assert(useQueryRealisedOutput.next());
+            auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
+            auto depId = DrvOutput { Hash::parseAnyPrefixed(depHash), depOutputName };
+            dependentRealisations.insert({depId, outputPath});
+        }
 
         return Ret{Realisation{
             .id = id,
             .outPath = outputPath,
             .signatures = signatures,
-            .drvOutputDeps = drvOutputDeps,
+            .dependentRealisations = dependentRealisations,
         }};
     });
 }
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 8793f9f4a..80ee15c49 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -254,25 +254,22 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         }});
 }
 
-std::set<DrvOutput> drvOutputReferences(
+std::map<DrvOutput, StorePath> drvOutputReferences(
     const std::set<Realisation> inputRealisations,
     const StorePathSet pathReferences)
 {
-    std::set<DrvOutput> res;
+    std::map<DrvOutput, StorePath> res;
 
-    std::map<StorePath, std::set<DrvOutput>> inputsByOutputPath;
-    for (const auto & input : inputRealisations)
-        inputsByOutputPath[input.outPath].insert(input.id);
-
-    for (const auto & path : pathReferences) {
-        auto theseInputs = inputsByOutputPath[path];
-        res.insert(theseInputs.begin(), theseInputs.end());
+    for (const auto & input : inputRealisations) {
+        if (pathReferences.count(input.outPath)) {
+            res.insert({input.id, input.outPath});
+        }
     }
 
     return res;
 }
 
-std::set<DrvOutput> drvOutputReferences(
+std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,
     const Derivation & drv,
     const StorePath & outputPath)
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index fab10f68c..d2d306476 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -33,7 +33,7 @@ void Realisation::closure(Store & store, std::set<Realisation> startOutputs, std
 {
     auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
         std::set<Realisation> res;
-        for (auto& currentDep : current.drvOutputDeps) {
+        for (auto& [currentDep, _] : current.dependentRealisations) {
             if (auto currentRealisation = store.queryRealisation(currentDep))
                 res.insert(*currentRealisation);
             else
@@ -60,14 +60,14 @@ void Realisation::closure(Store & store, std::set<Realisation> startOutputs, std
 }
 
 nlohmann::json Realisation::toJSON() const {
-    nlohmann::json jsonDrvOutputDeps;
-    for (auto & dep : drvOutputDeps)
-        jsonDrvOutputDeps.push_back(dep.to_string());
+    auto jsonDependentRealisations = nlohmann::json::object();
+    for (auto & [depId, depOutPath] : dependentRealisations)
+        jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string());
     return nlohmann::json{
         {"id", id.to_string()},
         {"outPath", outPath.to_string()},
         {"signatures", signatures},
-        {"drvOutputDeps", jsonDrvOutputDeps},
+        {"dependentRealisations", jsonDependentRealisations},
     };
 }
 
@@ -93,16 +93,16 @@ Realisation Realisation::fromJSON(
     if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
         signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
 
-    std::set <DrvOutput> drvOutputDeps;
-    if (auto jsonDependencies = json.find("drvOutputDeps"); jsonDependencies != json.end())
-        for (auto & jsonDep : *jsonDependencies)
-            drvOutputDeps.insert(DrvOutput::parse(jsonDep.get<std::string>()));
+    std::map <DrvOutput, StorePath> dependentRealisations;
+    if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end())
+        for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<std::map<std::string, std::string>>())
+            dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)});
 
     return Realisation{
         .id = DrvOutput::parse(getField("id")),
         .outPath = StorePath(getField("outPath")),
         .signatures = signatures,
-        .drvOutputDeps = drvOutputDeps,
+        .dependentRealisations = dependentRealisations,
     };
 }
 
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 776ab606c..8cda5a752 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -28,7 +28,13 @@ struct Realisation {
 
     StringSet signatures;
 
-    std::set<DrvOutput> drvOutputDeps;
+    /**
+     * The realisations that are required for the current one to be valid.
+     *
+     * When importing this realisation, the store will first check that all its
+     * dependencies exist, and map to the correct output path
+     */
+    std::map<DrvOutput, StorePath> dependentRealisations;
 
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 08573f709..6f29c430a 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -798,7 +798,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
             pool, Realisation::closure(*srcStore, toplevelRealisations),
             [&](const Realisation& current) -> std::set<Realisation> {
                 std::set<Realisation> children;
-                for (const auto& drvOutput : current.drvOutputDeps) {
+                for (const auto& [drvOutput, _] : current.dependentRealisations) {
                     auto currentChild = srcStore->queryRealisation(drvOutput);
                     if (!currentChild)
                         throw Error(
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 29af0f495..9a0027641 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -864,7 +864,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri)
 
 std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
 
-std::set<DrvOutput> drvOutputReferences(
+std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,
     const Derivation & drv,
     const StorePath & outputPath);

From a22755721b51949dbf03bc1eb156d6f37dc10bac Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 14:51:34 +0200
Subject: [PATCH 15/90] Recursively substitute the realisations

Make sure that whenever we substitute a realisation, we also substitute
its entire closure
---
 src/libstore/build/drv-output-substitution-goal.cc | 6 ++++++
 src/libutil/comparator.hh                          | 4 +++-
 tests/ca/substitute.sh                             | 8 ++++++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
index a5ac4c49d..1703e845d 100644
--- a/src/libstore/build/drv-output-substitution-goal.cc
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -53,6 +53,12 @@ void DrvOutputSubstitutionGoal::tryNext()
         return;
     }
 
+    for (const auto & [drvOutputDep, _] : outputInfo->dependentRealisations) {
+        if (drvOutputDep != id) {
+            addWaitee(worker.makeDrvOutputSubstitutionGoal(drvOutputDep));
+        }
+    }
+
     addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
 
     if (waitees.empty()) outPathValid();
diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh
index 0315dc506..eecd5b819 100644
--- a/src/libutil/comparator.hh
+++ b/src/libutil/comparator.hh
@@ -25,6 +25,8 @@
     }
 #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
 #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
 #define GENERATE_CMP(args...) \
     GENERATE_EQUAL(args) \
-    GENERATE_LEQ(args)
+    GENERATE_LEQ(args) \
+    GENERATE_NEQ(args)
diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh
index dfc4ea68e..c80feaacf 100644
--- a/tests/ca/substitute.sh
+++ b/tests/ca/substitute.sh
@@ -17,11 +17,15 @@ buildDrvs () {
 
 # Populate the remote cache
 clearStore
-buildDrvs --post-build-hook ../push-to-store.sh
+nix copy --to $REMOTE_STORE --file ./content-addressed.nix
 
 # Restart the build on an empty store, ensuring that we don't build
 clearStore
-buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA
+# Check that the thing we’ve just substituted has its realisation stored
+nix realisation info --file ./content-addressed.nix transitivelyDependentCA
+# Check that its dependencies have it too
+nix realisation info --file ./content-addressed.nix dependentCA rootCA
 
 # Same thing, but
 # 1. With non-ca derivations

From f35f9af78720cec098c5c2188bf34948b3a67c95 Mon Sep 17 00:00:00 2001
From: Keshav Kini <keshav.kini@gmail.com>
Date: Thu, 3 Jun 2021 17:59:39 -0700
Subject: [PATCH 16/90] Improve explanation of NIX_PATH prefix syntax

The previous wording seemed to imply that the "channel:" syntax would resolve to
a github archive URL, which is not the case.
---
 doc/manual/src/command-ref/env-common.md | 42 +++++++++++++-----------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md
index b709ca9d1..6e2403461 100644
--- a/doc/manual/src/command-ref/env-common.md
+++ b/doc/manual/src/command-ref/env-common.md
@@ -10,35 +10,39 @@ Most Nix commands interpret the following environment variables:
     A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e., `<path>`). For
     instance, the value
-    
+
         /home/eelco/Dev:/etc/nixos
-    
+
     will cause Nix to look for paths relative to `/home/eelco/Dev` and
     `/etc/nixos`, in this order. It is also possible to match paths
     against a prefix. For example, the value
-    
+
         nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
-    
+
     will cause Nix to search for `<nixpkgs/path>` in
     `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-    
+
     If a path in the Nix search path starts with `http://` or
     `https://`, it is interpreted as the URL of a tarball that will be
     downloaded and unpacked to a temporary location. The tarball must
     consist of a single top-level directory. For example, setting
     `NIX_PATH` to
-    
-        nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
-    
-    tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
-    channel.
-    
-    A following shorthand can be used to refer to the official channels:
-    
-        nixpkgs=channel:nixos-15.09
-    
-    The search path can be extended using the `-I` option, which takes
-    precedence over `NIX_PATH`.
+
+        nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
+
+    tells Nix to download and use the current contents of the
+    `master` branch in the `nixpkgs` repository.
+
+    The URLs of the tarballs from the official nixos.org channels (see
+    [the manual for `nix-channel`](nix-channel.md)) can be abbreviated
+    as `channel:<channel-name>`.  For instance, the following two
+    values of `NIX_PATH` are equivalent:
+
+        nixpkgs=channel:nixos-21.05
+        nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
+
+    The Nix search path can also be extended using the `-I` option to
+    many Nix commands, which takes precedence over `NIX_PATH`.
 
   - `NIX_IGNORE_SYMLINK_STORE`\
     Normally, the Nix store directory (typically `/nix/store`) is not
@@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables:
     builds are deployed to machines where `/nix/store` resolves
     differently. If you are sure that you’re not going to do that, you
     can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
-    
+
     Note that if you’re symlinking the Nix store so that you can put it
     on another file system than the root file system, on Linux you’re
     better off using `bind` mount points, e.g.,
@@ -59,7 +63,7 @@ Most Nix commands interpret the following environment variables:
     $ mkdir /nix
     $ mount -o bind /mnt/otherdisk/nix /nix
     ```
-    
+
     Consult the mount 8 manual page for details.
 
   - `NIX_STORE_DIR`\

From 7c077d2a0f4ccdeb84a4ad14ba5871a3fbe170e4 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Jun 2021 08:37:59 +0200
Subject: [PATCH 17/90] Add a ca-derivations required machine feature

Make ca-derivations require a `ca-derivations` machine feature, and
ca-aware builders expose it.

That way, a network of builders can mix ca-aware and non-ca-aware
machines, and the scheduler will send them in the right place.
---
 src/libstore/parsed-derivations.cc               |  2 ++
 src/libstore/store-api.cc                        |  7 +++++++
 src/libstore/store-api.hh                        |  4 +++-
 tests/build-remote-content-addressed-floating.sh |  2 ++
 tests/build-remote.sh                            | 11 ++++++++---
 5 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index c5c3ae3dc..5e383a9a4 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -91,6 +91,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
     StringSet res;
     for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
         res.insert(i);
+    if (!derivationHasKnownOutputPaths(drv.type()))
+        res.insert("ca-derivations");
     return res;
 }
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 93fcb068f..6ca8e5b24 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -337,6 +337,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     return info;
 }
 
+StringSet StoreConfig::getDefaultSystemFeatures()
+{
+    auto res = settings.systemFeatures.get();
+    if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+        res.insert("ca-derivations");
+    return res;
+}
 
 Store::Store(const Params & params)
     : StoreConfig(params)
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f66298991..9b4b10727 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -180,6 +180,8 @@ struct StoreConfig : public Config
 
     StoreConfig() = delete;
 
+    StringSet getDefaultSystemFeatures();
+
     virtual ~StoreConfig() { }
 
     virtual const std::string name() = 0;
@@ -196,7 +198,7 @@ struct StoreConfig : public Config
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
 
-    Setting<StringSet> systemFeatures{this, settings.systemFeatures,
+    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
         "system-features",
         "Optional features that the system this store builds on implements (like \"kvm\")."};
 
diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh
index 7447d92bd..13ef47d2d 100644
--- a/tests/build-remote-content-addressed-floating.sh
+++ b/tests/build-remote-content-addressed-floating.sh
@@ -4,4 +4,6 @@ file=build-hook-ca-floating.nix
 
 sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
 
+CONTENT_ADDRESSED=true
+
 source build-remote.sh
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 70f82e939..27d85a83d 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -6,12 +6,17 @@ unset NIX_STATE_DIR
 
 function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
 
+EXTRA_SYSTEM_FEATURES=()
+if [[ -n "$CONTENT_ADDRESSED" ]]; then
+    EXTRA_SYSTEM_FEATURES=("ca-derivations")
+fi
+
 builders=(
   # system-features will automatically be added to the outer URL, but not inner
   # remote-store URL.
-  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
-  "$TEST_ROOT/machine2 - - 1 1 bar"
-  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
+  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})"
+  "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})"
+  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})"
 )
 
 chmod -R +w $TEST_ROOT/machine* || true

From 7ac038fa4b83aaeff0b4dcf97b912cd7fd5f0ef6 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Jun 2021 09:24:24 +0200
Subject: [PATCH 18/90] Make `computeFSClosure` ca-aware

Fix #4820 by preventing nix-collect garbage from crashing if
`keep-outputs` or `keep-derivations` is true
---
 src/libstore/misc.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index bc5fd968c..3e54bfd8f 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -29,9 +29,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(i);
 
             if (includeDerivers && path.isDerivation())
-                for (auto& i : queryDerivationOutputs(path))
-                    if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                        res.insert(i);
+                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
+                    if (maybeOutPath && isValidPath(*maybeOutPath) && queryPathInfo(*maybeOutPath)->deriver == path)
+                        res.insert(*maybeOutPath);
             return res;
         };
     else
@@ -44,9 +44,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(ref);
 
             if (includeOutputs && path.isDerivation())
-                for (auto& i : queryDerivationOutputs(path))
-                    if (isValidPath(i))
-                        res.insert(i);
+                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
+                    if (maybeOutPath && isValidPath(*maybeOutPath))
+                        res.insert(*maybeOutPath);
 
             if (includeDerivers && info->deriver && isValidPath(*info->deriver))
                 res.insert(*info->deriver);

From 56605b468868b834e44a9700907b734428cb120a Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 20 May 2021 13:55:00 +0200
Subject: [PATCH 19/90] Make `nix-shell` support content-addressed derivations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Resolve the derivation before trying to load its environment −
essentially reproducing what the build loop does − so that we can
effectively access our dependencies (and not just their placeholders).

Fix #4821
---
 src/nix-build/nix-build.cc |  7 +++++++
 tests/ca/nix-shell.sh      | 10 ++++++++++
 tests/local.mk             |  1 +
 tests/nix-shell.sh         | 14 ++++++++++----
 tests/shell.nix            | 16 ++++++++++++++--
 5 files changed, 42 insertions(+), 6 deletions(-)
 create mode 100755 tests/ca/nix-shell.sh

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 9acbedda2..05eb7e234 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -387,6 +387,13 @@ static void main_nix_build(int argc, char * * argv)
 
         if (dryRun) return;
 
+        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+            auto resolvedDrv = drv.tryResolve(*store);
+            if (!resolvedDrv)
+                throw Error("unable to resolve the derivation '%s'. nix-shell can’t continue", drvInfo.queryDrvPath());
+            drv = *resolvedDrv;
+        }
+
         // Set the environment.
         auto env = getEnv();
 
diff --git a/tests/ca/nix-shell.sh b/tests/ca/nix-shell.sh
new file mode 100755
index 000000000..7f1a3a73e
--- /dev/null
+++ b/tests/ca/nix-shell.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+CONTENT_ADDRESSED=true
+cd ..
+source ./nix-shell.sh
+
diff --git a/tests/local.mk b/tests/local.mk
index 59eb4eb0f..d640bdd3c 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -47,6 +47,7 @@ nix_tests = \
   ca/build.sh \
   ca/substitute.sh \
   ca/signatures.sh \
+  ca/nix-shell.sh \
   ca/nix-run.sh \
   ca/nix-copy.sh
   # parallel.sh
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 4775bafb9..e3174dac1 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -2,6 +2,12 @@ source common.sh
 
 clearStore
 
+if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
+    nix-shell () {
+        command nix-shell --arg contentAddressed true "$@"
+    }
+fi
+
 # Test nix-shell -A
 export IMPURE_VAR=foo
 export SELECTED_IMPURE_VAR=baz
@@ -41,7 +47,7 @@ output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(f
 [ "$output" = "foo bar" ]
 
 # Test nix-shell shebang mode
-sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
 chmod a+rx $TEST_ROOT/shell.shebang.sh
 
 output=$($TEST_ROOT/shell.shebang.sh abc def)
@@ -49,7 +55,7 @@ output=$($TEST_ROOT/shell.shebang.sh abc def)
 
 # Test nix-shell shebang mode again with metacharacters in the filename.
 # First word of filename is chosen to not match any file in the test root.
-sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
@@ -58,7 +64,7 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
 # Test nix-shell shebang mode for ruby
 # This uses a fake interpreter that returns the arguments passed
 # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
-sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
 chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
@@ -66,7 +72,7 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 
 # Test nix-shell shebang mode for ruby again with metacharacters in the filename.
 # Note: fake interpreter only space-separates args without adding escapes to its output.
-sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
diff --git a/tests/shell.nix b/tests/shell.nix
index 24ebcc04c..53759f99a 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -1,6 +1,18 @@
-{ inNixShell ? false }:
+{ inNixShell ? false, contentAddressed ? false }:
 
-with import ./config.nix;
+let cfg = import ./config.nix; in
+with cfg;
+
+let
+  mkDerivation =
+    if contentAddressed then
+      args: cfg.mkDerivation ({
+        __contentAddressed = true;
+        outputHashMode = "recursive";
+        outputHashAlgo = "sha256";
+      } // args)
+    else cfg.mkDerivation;
+in
 
 let pkgs = rec {
   setupSh = builtins.toFile "setup" ''

From 2cf591a134f3ec6f634b47eeb522f422c64a9d33 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 11 Jun 2021 13:31:19 +0200
Subject: [PATCH 20/90] Make `nix develop` work with CA derivations

Fix #4823
---
 src/nix/develop.cc | 30 +++++++++++++++++++-----------
 tests/nix-shell.sh | 12 ++++++++++--
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 10f843651..aeeaf3e13 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -144,17 +144,26 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
-    for (auto & output : drv.outputs) {
-        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
-        drv.env[output.first] = "";
-    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
-    Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        for (auto & output : drv.outputs) {
+            output.second = {
+                .output = DerivationOutputDeferred{},
+            };
+            drv.env[output.first] = "";
+        }
+    } else {
+        for (auto & output : drv.outputs) {
+            output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+            drv.env[output.first] = "";
+        }
+        Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
 
-    for (auto & output : drv.outputs) {
-        auto outPath = store->makeOutputPath(output.first, h, drv.name);
-        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
-        drv.env[output.first] = store->printStorePath(outPath);
+        for (auto & output : drv.outputs) {
+            auto outPath = store->makeOutputPath(output.first, h, drv.name);
+            output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+            drv.env[output.first] = store->printStorePath(outPath);
+        }
     }
 
     auto shellDrvPath = writeDerivation(*store, drv);
@@ -162,8 +171,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Build the derivation. */
     store->buildPaths({DerivedPath::Built{shellDrvPath}});
 
-    for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
-        auto & [_1, optPath] = outputAndOptPath;
+    for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) {
         assert(optPath);
         auto & outPath = *optPath;
         assert(store->isValidPath(outPath));
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index e3174dac1..3481c2c69 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -6,6 +6,14 @@ if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
     nix-shell () {
         command nix-shell --arg contentAddressed true "$@"
     }
+
+    nix_develop() {
+        nix develop --arg contentAddressed true "$@"
+    }
+else
+    nix_develop() {
+        nix develop "$@"
+    }
 fi
 
 # Test nix-shell -A
@@ -79,13 +87,13 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
 [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ]
 
 # Test 'nix develop'.
-nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
 
 # Ensure `nix develop -c` preserves stdin
 echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
 
 # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal
-nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo
+nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
 
 # Test 'nix print-dev-env'.
 source <(nix print-dev-env -f shell.nix shellDrv)

From 96d7170e12c75acf363bf7a5a081a86b3c384384 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Sat, 12 Jun 2021 12:24:53 +0200
Subject: [PATCH 21/90] =?UTF-8?q?Don=E2=80=99t=20check=20the=20`deriver`?=
 =?UTF-8?q?=20field=20on=20computeFSClosure?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

That doesn’t really make sense with CA derivations (and wasn’t even
really correct before because of FO derivations, though that probably
didn’t matter much in practice)
---
 src/libstore/misc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 3e54bfd8f..48442563f 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -30,7 +30,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
 
             if (includeDerivers && path.isDerivation())
                 for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
-                    if (maybeOutPath && isValidPath(*maybeOutPath) && queryPathInfo(*maybeOutPath)->deriver == path)
+                    if (maybeOutPath && isValidPath(*maybeOutPath))
                         res.insert(*maybeOutPath);
             return res;
         };

From 79674c6cdb63a2e9421e38e47d5fe5b5c5ffbb32 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sat, 12 Jun 2021 23:50:26 -0500
Subject: [PATCH 22/90] Fix zsh completion script

Installed site-functions need to be run directly, not via compdef.
---
 misc/zsh/completion.zsh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh
index d4df6447e..a902e37dc 100644
--- a/misc/zsh/completion.zsh
+++ b/misc/zsh/completion.zsh
@@ -1,3 +1,5 @@
+#compdef nix
+
 function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
@@ -18,4 +20,4 @@ function _nix() {
   _describe 'nix' suggestions
 }
 
-compdef _nix nix
+_nix "$@"

From a3ce88725b44a30184cbe8706fc3a53a8610cc47 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 15 Jun 2021 12:11:31 +0200
Subject: [PATCH 23/90] Add a test for the gc with CA derivations

Also add a small architecture to easily run CA-enabled tests
---
 tests/ca/gc.sh      | 12 ++++++++++++
 tests/config.nix.in | 11 ++++++++++-
 tests/local.mk      |  1 +
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100755 tests/ca/gc.sh

diff --git a/tests/ca/gc.sh b/tests/ca/gc.sh
new file mode 100755
index 000000000..e4f9857d6
--- /dev/null
+++ b/tests/ca/gc.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Ensure that garbage collection works properly with ca derivations
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+
+cd ..
+source gc.sh
diff --git a/tests/config.nix.in b/tests/config.nix.in
index a57a8c596..9b00d9ddb 100644
--- a/tests/config.nix.in
+++ b/tests/config.nix.in
@@ -1,3 +1,12 @@
+let
+  contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1";
+  caArgs = if contentAddressedByDefault then {
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  } else {};
+in
+
 rec {
   shell = "@bash@";
 
@@ -15,4 +24,4 @@ rec {
       PATH = path;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
-}
+} // caArgs
diff --git a/tests/local.mk b/tests/local.mk
index 59eb4eb0f..e40178b80 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -2,6 +2,7 @@ nix_tests = \
   hash.sh lang.sh add.sh simple.sh dependencies.sh \
   config.sh \
   gc.sh \
+  ca/gc.sh \
   gc-concurrent.sh \
   gc-auto.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \

From 498677cbedf9c7dee1df86e775342cdf93a110bd Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 21 Jun 2021 14:16:13 +0200
Subject: [PATCH 24/90] Fix the remote build of CA derivations

Make sure that the derivation we send to the remote builder is exactly
the one that we want to build locally so that the output ids are exactly
the same

Fix #4845
---
 src/build-remote/build-remote.cc | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 57f2cd32d..0904f2ce9 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -277,7 +277,16 @@ connected:
 
         auto drv = store->readDerivation(*drvPath);
         auto outputHashes = staticOutputHashes(*store, drv);
-        drv.inputSrcs = store->parseStorePathSet(inputs);
+
+        // Hijack the inputs paths of the derivation to include all the paths
+        // that come from the `inputDrvs` set.
+        // We don’t do that for the derivations whose `inputDrvs` is empty
+        // because
+        // 1. It’s not needed
+        // 2. Changing the `inputSrcs` set changes the associated output ids,
+        //  which break CA derivations
+        if (!drv.inputDrvs.empty())
+            drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
 

From ce674cb2cf97fc14ce48acef7f8ef88f4ac771f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Mon, 21 Jun 2021 15:52:01 +0200
Subject: [PATCH 25/90] Properly set the output env variables

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/nix/develop.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index aeeaf3e13..350b08648 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -150,7 +150,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
             output.second = {
                 .output = DerivationOutputDeferred{},
             };
-            drv.env[output.first] = "";
+            drv.env[output.first] = hashPlaceholder(output.first);
         }
     } else {
         for (auto & output : drv.outputs) {

From dcabb461242e5c38e0a2780dc5869b91c13241d6 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 21 Jun 2021 16:28:06 +0200
Subject: [PATCH 26/90] Shorten a stupidly long sql query name

---
 src/libstore/local-store.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index debe70037..45bb5efc8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -59,8 +59,8 @@ struct LocalStore::State::Stmts {
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
-    SQLiteStmt QueryRealisationRealisationReferences;
-    SQLiteStmt AddRealisationRealisationReference;
+    SQLiteStmt QueryRealisationReferences;
+    SQLiteStmt AddRealisationReference;
 };
 
 int getSchema(Path schemaPath)
@@ -362,13 +362,13 @@ LocalStore::LocalStore(const Params & params)
                     where drvPath = ?
                     ;
             )");
-        state->stmts->QueryRealisationRealisationReferences.create(state->db,
+        state->stmts->QueryRealisationReferences.create(state->db,
             R"(
                 select drvPath, outputName from Realisations
                     join RealisationsRefs on realisationReference = Realisations.id
                     where referrer = ?;
             )");
-        state->stmts->AddRealisationRealisationReference.create(state->db,
+        state->stmts->AddRealisationReference.create(state->db,
             R"(
                 insert or replace into RealisationsRefs (referrer, realisationReference)
                 values (
@@ -719,7 +719,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
             .exec();
         uint64_t myId = state->db.getLastInsertedRowId();
         for (auto & [outputId, _] : info.dependentRealisations) {
-            state->stmts->AddRealisationRealisationReference
+            state->stmts->AddRealisationReference
                 .use()(myId)(outputId.strHash())(outputId.outputName)
                 .exec();
         }
@@ -1732,7 +1732,7 @@ std::optional<const Realisation> LocalStore::queryRealisation(
 
         std::map<DrvOutput, StorePath> dependentRealisations;
         auto useRealisationRefs(
-            state->stmts->QueryRealisationRealisationReferences.use()(
+            state->stmts->QueryRealisationReferences.use()(
                 realisationDbId));
         while (useRealisationRefs.next()) {
             auto depHash = useRealisationRefs.getStr(0);

From 3b58dbb3561e02dc94e88b531d0942a7f8ccf8ea Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 22 Jun 2021 11:29:55 +0200
Subject: [PATCH 27/90] nix-shell: Replace resolving failure error by an
 assertion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This shouldn’t happen in practice, so better make it explicit
---
 src/nix-build/nix-build.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 05eb7e234..3fec2c06c 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -389,8 +389,7 @@ static void main_nix_build(int argc, char * * argv)
 
         if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
             auto resolvedDrv = drv.tryResolve(*store);
-            if (!resolvedDrv)
-                throw Error("unable to resolve the derivation '%s'. nix-shell can’t continue", drvInfo.queryDrvPath());
+            assert(resolvedDrv && "Successfully resolved the derivation");
             drv = *resolvedDrv;
         }
 

From c13d7d0b9770741cc7093dc71ec8fc7978171f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Mon, 21 Jun 2021 16:37:45 +0200
Subject: [PATCH 28/90] Pass more values by reference

Rather than copying them around everywhere

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/misc.cc        | 4 ++--
 src/libstore/realisation.cc | 4 ++--
 src/libstore/realisation.hh | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 80ee15c49..96d73b70e 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -255,8 +255,8 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
 }
 
 std::map<DrvOutput, StorePath> drvOutputReferences(
-    const std::set<Realisation> inputRealisations,
-    const StorePathSet pathReferences)
+    const std::set<Realisation> & inputRealisations,
+    const StorePathSet & pathReferences)
 {
     std::map<DrvOutput, StorePath> res;
 
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index d2d306476..0d9d4b433 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -22,14 +22,14 @@ std::string DrvOutput::to_string() const {
     return strHash() + "!" + outputName;
 }
 
-std::set<Realisation> Realisation::closure(Store & store, std::set<Realisation> startOutputs)
+std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
 {
     std::set<Realisation> res;
     Realisation::closure(store, startOutputs, res);
     return res;
 }
 
-void Realisation::closure(Store & store, std::set<Realisation> startOutputs, std::set<Realisation> & res)
+void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
 {
     auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
         std::set<Realisation> res;
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 8cda5a752..7fdb65acd 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -44,8 +44,8 @@ struct Realisation {
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 
-    static std::set<Realisation> closure(Store &, std::set<Realisation>);
-    static void closure(Store &, std::set<Realisation>, std::set<Realisation>& res);
+    static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
+    static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res);
 
     StorePath getPath() const { return outPath; }
 

From 8d09a4f9a09473e6a32cc118ad10827ec5650700 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <regnat@users.noreply.github.com>
Date: Tue, 22 Jun 2021 09:31:25 +0200
Subject: [PATCH 29/90] Remove a useless string split

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libstore/store-api.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 6f29c430a..7e057b1ea 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -803,8 +803,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
                     if (!currentChild)
                         throw Error(
                             "Incomplete realisation closure: '%s' is a "
-                            "dependency "
-                            "of '%s' but isn’t registered",
+                            "dependency of '%s' but isn’t registered",
                             drvOutput.to_string(), current.id.to_string());
                     children.insert(*currentChild);
                 }

From 7c96a76dd7da69fa527dea7110d1156c1c5fbe9e Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 22 Jun 2021 09:26:55 +0200
Subject: [PATCH 30/90] Reformat the sql statements

---
 src/libstore/local-store.cc | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 45bb5efc8..b56df735c 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -713,14 +713,18 @@ void LocalStore::registerDrvOutput(const Realisation & info)
     settings.requireExperimentalFeature("ca-derivations");
     retrySQLite<void>([&]() {
         auto state(_state.lock());
-        state->stmts->RegisterRealisedOutput
-            .use()(info.id.strHash())(info.id.outputName)(printStorePath(
-                info.outPath))(concatStringsSep(" ", info.signatures))
+        state->stmts->RegisterRealisedOutput.use()
+                (info.id.strHash())
+                (info.id.outputName)
+                (printStorePath(info.outPath))
+                (concatStringsSep(" ", info.signatures))
             .exec();
         uint64_t myId = state->db.getLastInsertedRowId();
         for (auto & [outputId, _] : info.dependentRealisations) {
-            state->stmts->AddRealisationReference
-                .use()(myId)(outputId.strHash())(outputId.outputName)
+            state->stmts->AddRealisationReference.use()
+                (myId)
+                (outputId.strHash())
+                (outputId.outputName)
                 .exec();
         }
     });
@@ -1721,8 +1725,9 @@ std::optional<const Realisation> LocalStore::queryRealisation(
     typedef std::optional<const Realisation> Ret;
     return retrySQLite<Ret>([&]() -> Ret {
         auto state(_state.lock());
-        auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()(
-            id.strHash())(id.outputName));
+        auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()
+                (id.strHash())
+                (id.outputName));
         if (!useQueryRealisedOutput.next())
             return std::nullopt;
         auto realisationDbId = useQueryRealisedOutput.getInt(0);
@@ -1732,13 +1737,14 @@ std::optional<const Realisation> LocalStore::queryRealisation(
 
         std::map<DrvOutput, StorePath> dependentRealisations;
         auto useRealisationRefs(
-            state->stmts->QueryRealisationReferences.use()(
-                realisationDbId));
+            state->stmts->QueryRealisationReferences.use()
+                (realisationDbId));
         while (useRealisationRefs.next()) {
             auto depHash = useRealisationRefs.getStr(0);
             auto depOutputName = useRealisationRefs.getStr(1);
-            auto useQueryRealisedOutput(
-                state->stmts->QueryRealisedOutput.use()(depHash)(depOutputName));
+            auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()
+                    (depHash)
+                    (depOutputName));
             assert(useQueryRealisedOutput.next());
             auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
             auto depId = DrvOutput { Hash::parseAnyPrefixed(depHash), depOutputName };

From ed0e21a88d64d772304d079c39d76feca41c02d3 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 23 Jun 2021 07:45:41 +0200
Subject: [PATCH 31/90] Fix indentation

---
 src/libstore/local-store.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index b56df735c..348d964ca 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -714,10 +714,10 @@ void LocalStore::registerDrvOutput(const Realisation & info)
     retrySQLite<void>([&]() {
         auto state(_state.lock());
         state->stmts->RegisterRealisedOutput.use()
-                (info.id.strHash())
-                (info.id.outputName)
-                (printStorePath(info.outPath))
-                (concatStringsSep(" ", info.signatures))
+            (info.id.strHash())
+            (info.id.outputName)
+            (printStorePath(info.outPath))
+            (concatStringsSep(" ", info.signatures))
             .exec();
         uint64_t myId = state->db.getLastInsertedRowId();
         for (auto & [outputId, _] : info.dependentRealisations) {

From a5df669bc685834b16de0ab57723ff734c10d2f7 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 13:35:46 +0200
Subject: [PATCH 32/90] =?UTF-8?q?Add=20a=20test=20for=20the=20=E2=80=9Ctwo?=
 =?UTF-8?q?=20glibc=E2=80=9D=20issue?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/ca/duplicate-realisation-in-closure.sh | 26 +++++++++++++++
 tests/ca/nondeterministic.nix                | 35 ++++++++++++++++++++
 tests/local.mk                               |  1 +
 3 files changed, 62 insertions(+)
 create mode 100644 tests/ca/duplicate-realisation-in-closure.sh
 create mode 100644 tests/ca/nondeterministic.nix

diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/ca/duplicate-realisation-in-closure.sh
new file mode 100644
index 000000000..bfe2a4e08
--- /dev/null
+++ b/tests/ca/duplicate-realisation-in-closure.sh
@@ -0,0 +1,26 @@
+source ./common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
+export REMOTE_STORE="file://$REMOTE_STORE_DIR"
+
+rm -rf $REMOTE_STORE_DIR
+clearStore
+
+# Build dep1 and push that to the binary cache.
+# This entails building (and pushing) current-time.
+nix copy --to "$REMOTE_STORE" -f nondeterministic.nix dep1
+clearStore
+sleep 2 # To make sure that `$(date)` will be different
+# Build dep2.
+# As we’ve cleared the cache, we’ll have to rebuild current-time. And because
+# the current time isn’t the same as before, this will yield a new (different)
+# realisation
+nix build -f nondeterministic.nix dep2
+
+# Build something that depends both on dep1 and dep2.
+# If everything goes right, we should rebuild dep2 rather than fetch it from
+# the cache (because that would mean duplicating `current-time` in the closure),
+# and have `dep1 == dep2`.
+nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs
diff --git a/tests/ca/nondeterministic.nix b/tests/ca/nondeterministic.nix
new file mode 100644
index 000000000..d6d099a3e
--- /dev/null
+++ b/tests/ca/nondeterministic.nix
@@ -0,0 +1,35 @@
+with import ./config.nix;
+
+let mkCADerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+} // args);
+in
+
+rec {
+  currentTime = mkCADerivation {
+    name = "current-time";
+    buildCommand = ''
+      mkdir $out
+      echo $(date) > $out/current-time
+    '';
+  };
+  dep = seed: mkCADerivation {
+    name = "dep";
+    inherit seed;
+    buildCommand = ''
+      echo ${currentTime} > $out
+    '';
+  };
+  dep1 = dep 1;
+  dep2 = dep 2;
+  toplevel = mkCADerivation {
+    name = "toplevel";
+    buildCommand = ''
+      test ${dep1} == ${dep2}
+      touch $out
+    '';
+  };
+}
+
diff --git a/tests/local.mk b/tests/local.mk
index 0d594cda0..e7161d298 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -47,6 +47,7 @@ nix_tests = \
   compute-levels.sh \
   ca/build.sh \
   ca/build-with-garbage-path.sh \
+  ca/duplicate-realisation-in-closure.sh \
   ca/substitute.sh \
   ca/signatures.sh \
   ca/nix-run.sh \

From b8f7177a7b2e884cbfb8bbe3c1ee8d586159fbb3 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 16:19:46 +0200
Subject: [PATCH 33/90] Properly fail when trying to register an incoherent
 realisation

---
 src/libstore/local-store.cc | 136 +++++++++++++++++++++++++-----------
 src/libstore/local-store.hh |   2 +
 src/libstore/realisation.cc |   6 ++
 src/libstore/realisation.hh |   2 +
 4 files changed, 107 insertions(+), 39 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c2256635a..8df1d55b9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -53,6 +53,7 @@ struct LocalStore::State::Stmts {
     SQLiteStmt InvalidatePath;
     SQLiteStmt AddDerivationOutput;
     SQLiteStmt RegisterRealisedOutput;
+    SQLiteStmt UpdateRealisedOutput;
     SQLiteStmt QueryValidDerivers;
     SQLiteStmt QueryDerivationOutputs;
     SQLiteStmt QueryRealisedOutput;
@@ -345,6 +346,15 @@ LocalStore::LocalStore(const Params & params)
                 values (?, ?, (select id from ValidPaths where path = ?), ?)
                 ;
             )");
+        state->stmts->UpdateRealisedOutput.create(state->db,
+            R"(
+                update Realisations
+                    set signatures = ?
+                where
+                    drvPath = ? and
+                    outputName = ?
+                ;
+            )");
         state->stmts->QueryRealisedOutput.create(state->db,
             R"(
                 select Realisations.id, Output.path, Realisations.signatures from Realisations
@@ -710,14 +720,41 @@ void LocalStore::registerDrvOutput(const Realisation & info)
     settings.requireExperimentalFeature("ca-derivations");
     retrySQLite<void>([&]() {
         auto state(_state.lock());
-        state->stmts->RegisterRealisedOutput.use()
-            (info.id.strHash())
-            (info.id.outputName)
-            (printStorePath(info.outPath))
-            (concatStringsSep(" ", info.signatures))
-            .exec();
+        if (auto oldR = queryRealisation_(*state, info.id)) {
+            if (info.isCompatibleWith(*oldR)) {
+                auto combinedSignatures = oldR->signatures;
+                combinedSignatures.insert(info.signatures.begin(),
+                    info.signatures.end());
+                state->stmts->UpdateRealisedOutput.use()
+                    (concatStringsSep(" ", combinedSignatures))
+                    (info.id.strHash())
+                    (info.id.outputName)
+                    .exec();
+            } else {
+                throw Error("Trying to register a realisation of '%s', but we already "
+                            "have another one locally",
+                    info.id.to_string());
+            }
+        } else {
+            state->stmts->RegisterRealisedOutput.use()
+                (info.id.strHash())
+                (info.id.outputName)
+                (printStorePath(info.outPath))
+                (concatStringsSep(" ", info.signatures))
+                .exec();
+        }
         uint64_t myId = state->db.getLastInsertedRowId();
-        for (auto & [outputId, _] : info.dependentRealisations) {
+        for (auto & [outputId, depPath] : info.dependentRealisations) {
+            auto localRealisation = queryRealisationCore_(*state, outputId);
+            if (!localRealisation)
+                throw Error("unable to register the derivation '%s' as it "
+                            "depends on the non existent '%s'",
+                    info.id.to_string(), outputId.to_string());
+            if (localRealisation->second.outPath != depPath)
+                throw Error("unable to register the derivation '%s' as it "
+                            "depends on a realisation of '%s' that doesn’t"
+                            "match what we have locally",
+                    info.id.to_string(), outputId.to_string());
             state->stmts->AddRealisationReference.use()
                 (myId)
                 (outputId.strHash())
@@ -1734,46 +1771,67 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
     }
 }
 
-std::optional<const Realisation> LocalStore::queryRealisation(
-    const DrvOutput& id) {
-    typedef std::optional<const Realisation> Ret;
-    return retrySQLite<Ret>([&]() -> Ret {
-        auto state(_state.lock());
-        auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()
+std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_(
+        LocalStore::State & state,
+        const DrvOutput & id)
+{
+    auto useQueryRealisedOutput(
+            state.stmts->QueryRealisedOutput.use()
                 (id.strHash())
                 (id.outputName));
-        if (!useQueryRealisedOutput.next())
-            return std::nullopt;
-        auto realisationDbId = useQueryRealisedOutput.getInt(0);
-        auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
-        auto signatures =
-            tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
+    if (!useQueryRealisedOutput.next())
+        return std::nullopt;
+    auto realisationDbId = useQueryRealisedOutput.getInt(0);
+    auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
+    auto signatures =
+        tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
 
-        std::map<DrvOutput, StorePath> dependentRealisations;
-        auto useRealisationRefs(
-            state->stmts->QueryRealisationReferences.use()
-                (realisationDbId));
-        while (useRealisationRefs.next()) {
-            auto depHash = useRealisationRefs.getStr(0);
-            auto depOutputName = useRealisationRefs.getStr(1);
-            auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()
-                    (depHash)
-                    (depOutputName));
-            assert(useQueryRealisedOutput.next());
-            auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
-            auto depId = DrvOutput { Hash::parseAnyPrefixed(depHash), depOutputName };
-            dependentRealisations.insert({depId, outputPath});
-        }
-
-        return Ret{Realisation{
+    return {{
+        realisationDbId,
+        Realisation{
             .id = id,
             .outPath = outputPath,
             .signatures = signatures,
-            .dependentRealisations = dependentRealisations,
-        }};
-    });
+        }
+    }};
 }
 
+std::optional<const Realisation> LocalStore::queryRealisation_(
+            LocalStore::State & state,
+            const DrvOutput & id)
+{
+    auto maybeCore = queryRealisationCore_(state, id);
+    if (!maybeCore)
+        return std::nullopt;
+    auto [realisationDbId, res] = *maybeCore;
+
+    std::map<DrvOutput, StorePath> dependentRealisations;
+    auto useRealisationRefs(
+        state.stmts->QueryRealisationReferences.use()
+            (realisationDbId));
+    while (useRealisationRefs.next()) {
+        auto depId = DrvOutput {
+            Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
+            useRealisationRefs.getStr(1),
+        };
+        auto dependentRealisation = queryRealisationCore_(state, depId);
+        assert(dependentRealisation); // Enforced by the db schema
+        auto outputPath = dependentRealisation->second.outPath;
+        dependentRealisations.insert({depId, outputPath});
+    }
+
+    res.dependentRealisations = dependentRealisations;
+
+    return { res };
+}
+
+std::optional<const Realisation>
+LocalStore::queryRealisation(const DrvOutput &id) {
+  return retrySQLite<std::optional<const Realisation>>([&]() {
+    auto state(_state.lock());
+    return queryRealisation_(*state, id);
+  });
+}
 
 FixedOutputHash LocalStore::hashCAPath(
     const FileIngestionMethod & method, const HashType & hashType,
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 15c7fc306..a01d48c4b 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -203,6 +203,8 @@ public:
     void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
     void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
 
+    std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
+    std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
     std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
 
 private:
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index 0d9d4b433..76aec74ce 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -140,6 +140,12 @@ StorePath RealisedPath::path() const {
     return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }
 
+bool Realisation::isCompatibleWith(const Realisation & other) const
+{
+    assert (id == other.id);
+    return outPath == other.outPath;
+}
+
 void RealisedPath::closure(
     Store& store,
     const RealisedPath::Set& startPaths,
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 7fdb65acd..05d2bc44f 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -47,6 +47,8 @@ struct Realisation {
     static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
     static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res);
 
+    bool isCompatibleWith(const Realisation & other) const;
+
     StorePath getPath() const { return outPath; }
 
     GENERATE_CMP(Realisation, me->id, me->outPath);

From d32cf0c17a3e5e139c384375084d9eb6aa428c3b Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 19 May 2021 16:27:09 +0200
Subject: [PATCH 34/90] Gracefully ignore a substituter if it holds an
 incompatible realisation

---
 .../build/drv-output-substitution-goal.cc     | 22 ++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
index 1703e845d..ec3a8d758 100644
--- a/src/libstore/build/drv-output-substitution-goal.cc
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -17,6 +17,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
 void DrvOutputSubstitutionGoal::init()
 {
     trace("init");
+
+    /* If the derivation already exists, we’re done */
+    if (worker.store.queryRealisation(id)) {
+        amDone(ecSuccess);
+        return;
+    }
+
     subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
     tryNext();
 }
@@ -53,9 +60,18 @@ void DrvOutputSubstitutionGoal::tryNext()
         return;
     }
 
-    for (const auto & [drvOutputDep, _] : outputInfo->dependentRealisations) {
-        if (drvOutputDep != id) {
-            addWaitee(worker.makeDrvOutputSubstitutionGoal(drvOutputDep));
+    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
+        if (depId != id) {
+            if (auto localOutputInfo = worker.store.queryRealisation(depId);
+                localOutputInfo && localOutputInfo->outPath != depPath) {
+                warn(
+                    "substituter '%s' has an incompatible realisation for '%s', ignoring",
+                    sub->getUri(),
+                    depId.to_string());
+                tryNext();
+                return;
+            }
+            addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
         }
     }
 

From 40f925b2dacb481b62d325fb41641804524a5dc8 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 22 Jun 2021 10:42:23 +0200
Subject: [PATCH 35/90] Fix indentation

---
 src/libstore/local-store.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 8df1d55b9..064f7a432 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1826,11 +1826,12 @@ std::optional<const Realisation> LocalStore::queryRealisation_(
 }
 
 std::optional<const Realisation>
-LocalStore::queryRealisation(const DrvOutput &id) {
-  return retrySQLite<std::optional<const Realisation>>([&]() {
-    auto state(_state.lock());
-    return queryRealisation_(*state, id);
-  });
+LocalStore::queryRealisation(const DrvOutput & id)
+{
+    return retrySQLite<std::optional<const Realisation>>([&]() {
+        auto state(_state.lock());
+        return queryRealisation_(*state, id);
+    });
 }
 
 FixedOutputHash LocalStore::hashCAPath(

From 16fb7d8d95a8bc81e7df885ab4167c8a03f1dddf Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 22 Jun 2021 10:46:29 +0200
Subject: [PATCH 36/90] Display the diverging paths in case of a realisation
 mismatch

---
 src/libstore/build/drv-output-substitution-goal.cc | 9 +++++++--
 src/libstore/local-store.cc                        | 9 +++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
index ec3a8d758..be270d079 100644
--- a/src/libstore/build/drv-output-substitution-goal.cc
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -65,9 +65,14 @@ void DrvOutputSubstitutionGoal::tryNext()
             if (auto localOutputInfo = worker.store.queryRealisation(depId);
                 localOutputInfo && localOutputInfo->outPath != depPath) {
                 warn(
-                    "substituter '%s' has an incompatible realisation for '%s', ignoring",
+                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
+                    "Local:  %s\n"
+                    "Remote: %s",
                     sub->getUri(),
-                    depId.to_string());
+                    depId.to_string(),
+                    worker.store.printStorePath(localOutputInfo->outPath),
+                    worker.store.printStorePath(depPath)
+                );
                 tryNext();
                 return;
             }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 064f7a432..d7c7f8e1d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -732,8 +732,13 @@ void LocalStore::registerDrvOutput(const Realisation & info)
                     .exec();
             } else {
                 throw Error("Trying to register a realisation of '%s', but we already "
-                            "have another one locally",
-                    info.id.to_string());
+                            "have another one locally.\n"
+                            "Local:  %s\n"
+                            "Remote: %s",
+                    info.id.to_string(),
+                    printStorePath(oldR->outPath),
+                    printStorePath(info.outPath)
+                );
             }
         } else {
             state->stmts->RegisterRealisedOutput.use()

From c878cee8954151aaa1054af7ef3746a979b05832 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 22 Jun 2021 10:50:28 +0200
Subject: [PATCH 37/90] Assert that compatible realisations have the same
 dependencies
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Should always hold, but that’s not necessarily obvious, so better
enforce it
---
 src/libstore/realisation.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index 76aec74ce..eadec594c 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -143,7 +143,11 @@ StorePath RealisedPath::path() const {
 bool Realisation::isCompatibleWith(const Realisation & other) const
 {
     assert (id == other.id);
-    return outPath == other.outPath;
+    if (outPath == other.outPath) {
+        assert(dependentRealisations == other.dependentRealisations);
+        return true;
+    }
+    return false;
 }
 
 void RealisedPath::closure(

From 4f80464645e4c8e7ca9455fc53cc76dc50f688ed Mon Sep 17 00:00:00 2001
From: Alyssa Ross <hi@alyssa.is>
Date: Tue, 1 Jun 2021 07:58:21 +0000
Subject: [PATCH 38/90] Apply OS checks to host platform, not build

Previously, the build system used uname(1) output when it wanted to
check the operating system it was being built for, which meant that it
didn't take into-account cross-compilation when the build and host
operating systems were different.

To fix this, instead of consulting uname output, we consult the host
triple, specifically the third "kernel" part.

For "kernel"s with stable ABIs, like Linux or Cygwin, we can use a
simple ifeq to test whether we're compiling for that system, but for
other platforms, like Darwin, FreeBSD, or Solaris, we have to use a
more complicated check to take into account the version numbers at the
end of the "kernel"s.  I couldn't find a way to just strip these
version numbers in GNU Make without shelling out, which would be even
more ugly IMO.  Because these checks differ between kernels, and the
patsubst ones are quite fiddly, I've added variables for each host OS
we might want to check to make them easier to reuse.
---
 Makefile.config.in                       |  1 +
 misc/launchd/local.mk                    |  2 +-
 misc/systemd/local.mk                    |  2 +-
 misc/upstart/local.mk                    |  2 +-
 mk/lib.mk                                | 27 +++++++++++++++++++-----
 mk/libraries.mk                          | 18 ++++++++--------
 nix-rust/local.mk                        |  8 +++----
 perl/Makefile.config.in                  |  1 +
 perl/configure.ac                        |  2 ++
 perl/local.mk                            |  2 +-
 src/libexpr/local.mk                     |  2 +-
 src/libstore/local.mk                    |  6 +++---
 src/resolve-system-dependencies/local.mk |  2 +-
 13 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index fd92365eb..c8c4446b4 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -1,3 +1,4 @@
+HOST_OS = @host_os@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
 BOOST_LDFLAGS = @BOOST_LDFLAGS@
diff --git a/misc/launchd/local.mk b/misc/launchd/local.mk
index 0ba722efb..a39188fe6 100644
--- a/misc/launchd/local.mk
+++ b/misc/launchd/local.mk
@@ -1,4 +1,4 @@
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 
   $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons))
 
diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk
index 785db52a4..1fa037485 100644
--- a/misc/systemd/local.mk
+++ b/misc/systemd/local.mk
@@ -1,4 +1,4 @@
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
 
diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk
index 5071676dc..2fbfb29b9 100644
--- a/misc/upstart/local.mk
+++ b/misc/upstart/local.mk
@@ -1,4 +1,4 @@
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644)))
 
diff --git a/mk/lib.mk b/mk/lib.mk
index 975102531..92f0983d5 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -10,8 +10,25 @@ bin-scripts :=
 noinst-scripts :=
 man-pages :=
 install-tests :=
-OS = $(shell uname -s)
 
+ifdef HOST_OS
+  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
+  ifeq ($(HOST_KERNEL), cygwin)
+    HOST_CYGWIN = 1
+  endif
+  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
+    HOST_DARWIN = 1
+  endif
+  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
+    HOST_FREEBSD = 1
+  endif
+  ifeq ($(HOST_KERNEL), linux)
+    HOST_LINUX = 1
+  endif
+  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
+    HOST_SOLARIS = 1
+  endif
+endif
 
 # Hack to define a literal space.
 space :=
@@ -50,16 +67,16 @@ endif
 BUILD_SHARED_LIBS ?= 1
 
 ifeq ($(BUILD_SHARED_LIBS), 1)
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
     GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
   else
     GLOBAL_CFLAGS += -fPIC
     GLOBAL_CXXFLAGS += -fPIC
   endif
-  ifneq ($(OS), Darwin)
-   ifneq ($(OS), SunOS)
-    ifneq ($(OS), FreeBSD)
+  ifndef HOST_DARWIN
+   ifndef HOST_SOLARIS
+    ifndef HOST_FREEBSD
      GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries
     endif
    endif
diff --git a/mk/libraries.mk b/mk/libraries.mk
index 7c0e4f100..07bd54dab 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -1,9 +1,9 @@
 libs-list :=
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
   SO_EXT = dylib
 else
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     SO_EXT = dll
   else
     SO_EXT = so
@@ -59,7 +59,7 @@ define build-library
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH))
 
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     $(1)_INSTALL_DIR ?= $$(bindir)
   else
     $(1)_INSTALL_DIR ?= $$(libdir)
@@ -73,18 +73,18 @@ define build-library
   ifeq ($(BUILD_SHARED_LIBS), 1)
 
     ifdef $(1)_ALLOW_UNDEFINED
-      ifeq ($(OS), Darwin)
+      ifdef HOST_DARWIN
         $(1)_LDFLAGS += -undefined suppress -flat_namespace
       endif
     else
-      ifneq ($(OS), Darwin)
-        ifneq (CYGWIN,$(findstring CYGWIN,$(OS)))
+      ifndef HOST_DARWIN
+        ifndef HOST_CYGWIN
           $(1)_LDFLAGS += -Wl,-z,defs
         endif
       endif
     endif
 
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       $(1)_LDFLAGS += -Wl,-soname=$$($(1)_NAME).$(SO_EXT)
     endif
 
@@ -93,7 +93,7 @@ define build-library
     $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
 	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
     endif
     $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
@@ -108,7 +108,7 @@ define build-library
 	$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       ifeq ($(SET_RPATH_TO_LIBS), 1)
         $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR)
       else
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index 9650cdf93..538244594 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -11,12 +11,12 @@ libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
 libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust
 libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust
 
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 libnixrust_LDFLAGS_USE += -ldl
 libnixrust_LDFLAGS_USE_INSTALLED += -ldl
 endif
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
 else
 libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
@@ -31,7 +31,7 @@ $(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
 
 $(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
 	$(target-gen) cp $^ $@
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 	install_name_tool -id $@ $@
 endif
 
@@ -40,7 +40,7 @@ clean: clean-rust
 clean-rust:
 	$(suppress) rm -rfv nix-rust/target
 
-ifneq ($(OS), Darwin)
+ifndef HOST_DARWIN
 check: rust-tests
 
 rust-tests:
diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in
index eccfbd9f6..d856de3ad 100644
--- a/perl/Makefile.config.in
+++ b/perl/Makefile.config.in
@@ -1,3 +1,4 @@
+HOST_OS = @host_os@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
diff --git a/perl/configure.ac b/perl/configure.ac
index 85183c005..eb65ac17b 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -7,6 +7,8 @@ CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 
+AC_CANONICAL_HOST
+
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
 
diff --git a/perl/local.mk b/perl/local.mk
index b13d4c0d6..0eae651d8 100644
--- a/perl/local.mk
+++ b/perl/local.mk
@@ -28,7 +28,7 @@ Store_CXXFLAGS = \
 
 Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS)
 
-ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+ifdef HOST_CYGWIN
   archlib = $(shell perl -E 'use Config; print $$Config{archlib};')
   libperl = $(shell perl -E 'use Config; print $$Config{libperl};')
   Store_LDFLAGS += $(shell find ${archlib} -name ${libperl})
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index c40abfb78..1aed8e152 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS = -lboost_context
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index b6652984c..2fc334a82 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -9,11 +9,11 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 libstore_LIBS = libutil
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
  libstore_LDFLAGS += -ldl
 endif
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 endif
 
@@ -23,7 +23,7 @@ ifeq ($(ENABLE_S3), 1)
 	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
 endif
 
-ifeq ($(OS), SunOS)
+ifdef HOST_SOLARIS
 	libstore_LDFLAGS += -lsocket
 endif
 
diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk
index 054ae01cb..fc48a8417 100644
--- a/src/resolve-system-dependencies/local.mk
+++ b/src/resolve-system-dependencies/local.mk
@@ -1,4 +1,4 @@
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
   programs += resolve-system-dependencies
 endif
 

From fd3f5e9085737734d8769c21559461c26dc5a165 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 23 Jun 2021 17:36:50 +0200
Subject: [PATCH 39/90] Make the CA tests actuall test CA derivations

Fix a mistake in config.nix that was preventing
`NIX_TESTS_CA_BY_DEFAULT` from having any meaningful effect
---
 tests/config.nix.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/config.nix.in b/tests/config.nix.in
index 9b00d9ddb..7facbdcbc 100644
--- a/tests/config.nix.in
+++ b/tests/config.nix.in
@@ -22,6 +22,6 @@ rec {
       builder = shell;
       args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
-    } // removeAttrs args ["builder" "meta"])
+    } // caArgs // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
-} // caArgs
+}

From 01a3f4d7ec79d434dc34b64a761ddf681fe63a48 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 23 Jun 2021 17:37:29 +0200
Subject: [PATCH 40/90] Fix the CA gc test

Broken by https://github.com/NixOS/nix/issues/4936
---
 tests/gc.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/gc.sh b/tests/gc.sh
index 8b4f8d282..cf0e2c32d 100644
--- a/tests/gc.sh
+++ b/tests/gc.sh
@@ -12,7 +12,7 @@ ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo
 nix-store --gc --print-roots | grep $outPath
 nix-store --gc --print-live | grep $outPath
 nix-store --gc --print-dead | grep $drvPath
-if nix-store --gc --print-dead | grep $outPath; then false; fi
+if nix-store --gc --print-dead | grep -E $outPath$; then false; fi
 
 nix-store --gc --print-dead
 

From bee71e1bb1c63dc730e31523dd8075c922153051 Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Wed, 2 Jun 2021 00:45:03 +0200
Subject: [PATCH 41/90] Add a fish completion script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is only rudimentary support as allowed by `NIX_GET_COMPLETIONS`.

In the future, we could use complete’s `--wraps` argument to autocomplete arguments for programs after `nix shell -c`.
---
 Makefile                  |  1 +
 misc/fish/completion.fish | 37 +++++++++++++++++++++++++++++++++++++
 misc/fish/local.mk        |  1 +
 3 files changed, 39 insertions(+)
 create mode 100644 misc/fish/completion.fish
 create mode 100644 misc/fish/local.mk

diff --git a/Makefile b/Makefile
index b7f0e79db..dd259e5cd 100644
--- a/Makefile
+++ b/Makefile
@@ -12,6 +12,7 @@ makefiles = \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \
+  misc/fish/local.mk \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
diff --git a/misc/fish/completion.fish b/misc/fish/completion.fish
new file mode 100644
index 000000000..bedbefaf8
--- /dev/null
+++ b/misc/fish/completion.fish
@@ -0,0 +1,37 @@
+function _nix_complete
+  # Get the current command up to a cursor.
+  # - Behaves correctly even with pipes and nested in commands like env.
+  # - TODO: Returns the command verbatim (does not interpolate variables).
+  #   That might not be optimal for arguments like -f.
+  set -l nix_args (commandline --current-process --tokenize --cut-at-cursor)
+  # --cut-at-cursor with --tokenize removes the current token so we need to add it separately.
+  # https://github.com/fish-shell/fish-shell/issues/7375
+  # Can be an empty string.
+  set -l current_token (commandline --current-token --cut-at-cursor)
+
+  # Nix wants the index of the argv item to complete but the $nix_args variable
+  # also contains the program name (argv[0]) so we would need to subtract 1.
+  # But the variable also misses the current token so it cancels out.
+  set -l nix_arg_to_complete (count $nix_args)
+
+  env NIX_GET_COMPLETIONS=$nix_arg_to_complete $nix_args $current_token
+end
+
+function _nix_accepts_files
+  set -l response (_nix_complete)
+  # First line is either filenames or no-filenames.
+  test $response[1] = 'filenames'
+end
+
+function _nix
+  set -l response (_nix_complete)
+  # Skip the first line since it handled by _nix_accepts_files.
+  # Tail lines each contain a command followed by a tab character and, optionally, a description.
+  # This is also the format fish expects.
+  string collect -- $response[2..-1]
+end
+
+# Disable file path completion if paths do not belong in the current context.
+complete --command nix --condition 'not _nix_accepts_files' --no-files
+
+complete --command nix --arguments '(_nix)'
diff --git a/misc/fish/local.mk b/misc/fish/local.mk
new file mode 100644
index 000000000..ece899fc3
--- /dev/null
+++ b/misc/fish/local.mk
@@ -0,0 +1 @@
+$(eval $(call install-file-as, $(d)/completion.fish, $(datarootdir)/fish/vendor_completions.d/nix.fish, 0644))

From e3d11f9a9ca8cf1663de0182154683db250df095 Mon Sep 17 00:00:00 2001
From: Thomas Churchman <thomas@kepow.org>
Date: Wed, 23 Jun 2021 22:07:55 +0100
Subject: [PATCH 42/90] Improve machine store URI parsing

---
 src/libstore/machines.cc | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index b42e5e434..9843ccf04 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -16,13 +16,18 @@ Machine::Machine(decltype(storeUri) storeUri,
     decltype(mandatoryFeatures) mandatoryFeatures,
     decltype(sshPublicHostKey) sshPublicHostKey) :
     storeUri(
-        // Backwards compatibility: if the URI is a hostname,
-        // prepend ssh://.
+        // Backwards compatibility: if the URI is schemeless, is not a path,
+        // and is not one of the special store connection words, prepend
+        // ssh://.
         storeUri.find("://") != std::string::npos
-        || hasPrefix(storeUri, "local")
-        || hasPrefix(storeUri, "remote")
-        || hasPrefix(storeUri, "auto")
-        || hasPrefix(storeUri, "/")
+        || storeUri.find("/") != std::string::npos
+        || storeUri == "auto"
+        || storeUri == "daemon"
+        || storeUri == "local"
+        || hasPrefix(storeUri, "auto?")
+        || hasPrefix(storeUri, "daemon?")
+        || hasPrefix(storeUri, "local?")
+        || hasPrefix(storeUri, "?")
         ? storeUri
         : "ssh://" + storeUri),
     systemTypes(systemTypes),

From be7a4a6a1303ddbd1f8942f02c04f69016c51da0 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 24 Jun 2021 11:41:51 +0200
Subject: [PATCH 43/90] Make the post-build-hook also run for unresolved CA
 derivations

Fix #4837
---
 src/libstore/build/derivation-goal.cc | 127 +++++++++++++++-----------
 tests/ca/post-hook.sh                 |  11 +++
 tests/local.mk                        |   1 +
 tests/post-hook.sh                    |   2 +-
 4 files changed, 89 insertions(+), 52 deletions(-)
 create mode 100755 tests/ca/post-hook.sh

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 8c9ef0101..73d1ed7cc 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -739,6 +739,63 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 {
 }
 
+void runPostBuildHook(
+    Store & store,
+    Logger & logger,
+    const StorePath & drvPath,
+    StorePathSet outputPaths
+)
+{
+    auto hook = settings.postBuildHook;
+    if (hook == "")
+        return;
+
+    Activity act(logger, lvlInfo, actPostBuildHook,
+            fmt("running post-build-hook '%s'", settings.postBuildHook),
+            Logger::Fields{store.printStorePath(drvPath)});
+    PushActivity pact(act.id);
+    std::map<std::string, std::string> hookEnvironment = getEnv();
+
+    hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
+    hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
+
+    RunOptions opts(settings.postBuildHook, {});
+    opts.environment = hookEnvironment;
+
+    struct LogSink : Sink {
+        Activity & act;
+        std::string currentLine;
+
+        LogSink(Activity & act) : act(act) { }
+
+        void operator() (std::string_view data) override {
+            for (auto c : data) {
+                if (c == '\n') {
+                    flushLine();
+                } else {
+                    currentLine += c;
+                }
+            }
+        }
+
+        void flushLine() {
+            act.result(resPostBuildLogLine, currentLine);
+            currentLine.clear();
+        }
+
+        ~LogSink() {
+            if (currentLine != "") {
+                currentLine += '\n';
+                flushLine();
+            }
+        }
+    };
+    LogSink sink(act);
+
+    opts.standardOut = &sink;
+    opts.mergeStderrToStdout = true;
+    runProgram2(opts);
+}
 
 void DerivationGoal::buildDone()
 {
@@ -804,57 +861,15 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (std::string_view data) override {
-                    for (auto c : data) {
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
+        StorePathSet outputPaths;
+        for (auto & [_, path] : finalOutputs)
+            outputPaths.insert(path);
+        runPostBuildHook(
+            worker.store,
+            *logger,
+            drvPath,
+            outputPaths
+        );
 
         if (buildMode == bmCheck) {
             cleanupPostOutputsRegisteredModeCheck();
@@ -910,6 +925,8 @@ void DerivationGoal::resolvedFinished() {
 
     auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
 
+    StorePathSet outputPaths;
+
     // `wantedOutputs` might be empty, which means “all the outputs”
     auto realWantedOutputs = wantedOutputs;
     if (realWantedOutputs.empty())
@@ -930,6 +947,7 @@ void DerivationGoal::resolvedFinished() {
             newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
+            outputPaths.insert(realisation->outPath);
         } else {
             // If we don't have a realisation, then it must mean that something
             // failed when building the resolved drv
@@ -937,6 +955,13 @@ void DerivationGoal::resolvedFinished() {
         }
     }
 
+    runPostBuildHook(
+        worker.store,
+        *logger,
+        drvPath,
+        outputPaths
+    );
+
     // This is potentially a bit fishy in terms of error reporting. Not sure
     // how to do it in a cleaner way
     amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
diff --git a/tests/ca/post-hook.sh b/tests/ca/post-hook.sh
new file mode 100755
index 000000000..4b8da4cd8
--- /dev/null
+++ b/tests/ca/post-hook.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+cd ..
+source ./post-hook.sh
+
+
diff --git a/tests/local.mk b/tests/local.mk
index 82cec1df3..35ee9e271 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -39,6 +39,7 @@ nix_tests = \
   search.sh \
   nix-copy-ssh.sh \
   post-hook.sh \
+  ca/post-hook.sh \
   function-trace.sh \
   recursive.sh \
   describe-stores.sh \
diff --git a/tests/post-hook.sh b/tests/post-hook.sh
index aa3e6a574..238a8f826 100644
--- a/tests/post-hook.sh
+++ b/tests/post-hook.sh
@@ -4,7 +4,7 @@ clearStore
 
 rm -f $TEST_ROOT/result
 
-export REMOTE_STORE=$TEST_ROOT/remote_store
+export REMOTE_STORE=file:$TEST_ROOT/remote_store
 
 # Build the dependencies and push them to the remote store
 nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh

From bec83a6f953ab045e85d50b0cab9de176cd00bc0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 16 Nov 2020 11:03:53 +0100
Subject: [PATCH 44/90] BoehmGCStackAllocator: ignore stack protection page

This fixes a crash that looks like:

```
Thread 1 "nix-build" received signal SIGSEGV, Segmentation fault.
0x00007ffff7ad22a0 in GC_push_all_eager () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
(gdb) bt
0  0x00007ffff7ad22a0 in GC_push_all_eager () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
1  0x00007ffff7adeefb in GC_push_all_stacks () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
2  0x00007ffff7ad5ac7 in GC_mark_some () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
3  0x00007ffff7ad77bd in GC_stopped_mark () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
4  0x00007ffff7adbe3a in GC_try_to_collect_inner.part.0 () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
5  0x00007ffff7adc2a2 in GC_collect_or_expand () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
6  0x00007ffff7adc4f8 in GC_allocobj () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
7  0x00007ffff7adc88f in GC_generic_malloc_inner () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
8  0x00007ffff7ae1a04 in GC_generic_malloc_many () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
9  0x00007ffff7ae1c72 in GC_malloc_kind () from /nix/store/p1z58l18klf88iijpd0qi8yd2n9lhlk4-boehm-gc-8.0.4/lib/libgc.so.1
10 0x00007ffff7e003d6 in nix::EvalState::allocValue() () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
11 0x00007ffff7e04b9c in nix::EvalState::callPrimOp(nix::Value&, nix::Value&, nix::Value&, nix::Pos const&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
12 0x00007ffff7e0a773 in nix::EvalState::callFunction(nix::Value&, nix::Value&, nix::Value&, nix::Pos const&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
13 0x00007ffff7e0a91d in nix::ExprApp::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
14 0x00007ffff7e0a8f8 in nix::ExprApp::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
15 0x00007ffff7e0e0e8 in nix::ExprOpNEq::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
16 0x00007ffff7e0d708 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
17 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
18 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
19 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
20 0x00007ffff7e0d695 in nix::ExprOpOr::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
21 0x00007ffff7e09e19 in nix::ExprOpNot::eval(nix::EvalState&, nix::Env&, nix::Value&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
22 0x00007ffff7e0a792 in nix::EvalState::callFunction(nix::Value&, nix::Value&, nix::Value&, nix::Pos const&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
23 0x00007ffff7e8cba0 in nix::addPath(nix::EvalState&, nix::Pos const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, nix::Value*, nix::FileIngestionMethod, std::optional<nix::Hash>, nix::Value&)::{lambda(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)#1}::operator()(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixexpr.so
24 0x00007ffff752e6f9 in nix::dump(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, nix::Sink&, std::function<bool (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so
25 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, nix::Sink&, std::function<bool (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so
26 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, nix::Sink&, std::function<bool (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so
27 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, nix::Sink&, std::function<bool (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so
28 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, nix::Sink&, std::function<bool (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so
29 0x00007ffff752e8e2 in nix::dump(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, nix::Sink&, std::function<bool (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>&) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so
30 0x00007ffff757f8c0 in void boost::context::detail::fiber_entry<boost::context::detail::fiber_record<boost::context::fiber, nix::VirtualStackAllocator, boost::coroutines2::detail::pull_coroutine<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >::control_block::control_block<nix::VirtualStackAllocator, nix::sinkToSource(std::function<void (nix::Sink&)>, std::function<void ()>)::SinkToSource::read(char*, unsigned long)::{lambda(boost::coroutines2::detail::push_coroutine<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >&)#1}>(boost::context::preallocated, nix::VirtualStackAllocator&&, nix::sinkToSource(std::function<void (nix::Sink&)>, std::function<void ()>)::SinkToSource::read(char*, unsigned long)::{lambda(boost::coroutines2::detail::push_coroutine<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >&)#1}&&)::{lambda(boost::context::fiber&&)#1}> >(boost::context::detail::transfer_t) () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libnixutil.so
31 0x00007ffff6f331ef in make_fcontext () from /nix/store/hzdzcv9d3bc8rlsaphh7x54zsf0x8nx6-nix-2.4pre20210601_5985b8b/lib/libboost_context.so.1.69.0
32 0x0000000000000000 in ?? ()
```
---
 src/libexpr/eval.cc | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index ef9f8efca..c078bf4a1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -233,22 +233,34 @@ static void * oomHandler(size_t requested)
 }
 
 class BoehmGCStackAllocator : public StackAllocator {
-  boost::coroutines2::protected_fixedsize_stack stack {
-    // We allocate 8 MB, the default max stack size on NixOS.
-    // A smaller stack might be quicker to allocate but reduces the stack
-    // depth available for source filter expressions etc.
-    std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
+    boost::coroutines2::protected_fixedsize_stack stack {
+        // We allocate 8 MB, the default max stack size on NixOS.
+        // A smaller stack might be quicker to allocate but reduces the stack
+        // depth available for source filter expressions etc.
+        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
     };
 
+    // This is specific to boost::coroutines2::protected_fixedsize_stack.
+    // The stack protection page is included in sctx.size, so we have to
+    // subtract one page size from the stack size.
+    std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) {
+        return sctx.size - boost::context::stack_traits::page_size();
+    }
+
   public:
     boost::context::stack_context allocate() override {
         auto sctx = stack.allocate();
-        GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+
+        // Stacks generally start at a high address and grow to lower addresses.
+        // Architectures that do the opposite are rare; in fact so rare that
+        // boost_routine does not implement it.
+        // So we subtract the stack size.
+        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
         return sctx;
     }
 
     void deallocate(boost::context::stack_context sctx) override {
-        GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
         stack.deallocate(sctx);
     }
 

From 7746cb13dc4e644c9792b3e3666cd49635d694e0 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 23 Jun 2021 17:27:18 +0200
Subject: [PATCH 45/90] Make CA derivations compatible with recursive Nix

Add an access-control list to the realisations in recursive-nix (similar
to the already existing one for store paths), so that we can build
content-addressed derivations in the restricted store.

Fix #4353
---
 src/libstore/build/local-derivation-goal.cc | 21 +++++++++++++++++++--
 src/libstore/build/local-derivation-goal.hh |  8 ++++++++
 tests/ca/recursive.sh                       | 11 +++++++++++
 tests/local.mk                              |  1 +
 tests/recursive.sh                          | 17 +++++++++--------
 5 files changed, 48 insertions(+), 10 deletions(-)
 create mode 100755 tests/ca/recursive.sh

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 279139020..ba0aca29c 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1333,13 +1333,18 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
     // XXX: This should probably be allowed if the realisation corresponds to
     // an allowed derivation
-    { throw Error("queryRealisation"); }
+    {
+        if (!goal.isAllowed(id))
+            throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
+        return next->queryRealisation(id);
+    }
 
     void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
         StorePathSet newPaths;
+        std::set<Realisation> newRealisations;
 
         for (auto & req : paths) {
             if (!goal.isAllowed(req))
@@ -1352,16 +1357,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             auto p =  std::get_if<DerivedPath::Built>(&path);
             if (!p) continue;
             auto & bfd = *p;
+            auto drv = readDerivation(bfd.drvPath);
+            auto drvHashes = staticOutputHashes(*this, drv);
             auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
             for (auto & [outputName, outputPath] : outputs)
-                if (wantOutput(outputName, bfd.outputs))
+                if (wantOutput(outputName, bfd.outputs)) {
                     newPaths.insert(outputPath);
+                    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                        auto thisRealisation = next->queryRealisation(
+                            DrvOutput{drvHashes.at(outputName), outputName}
+                        );
+                        assert(thisRealisation);
+                        newRealisations.insert(*thisRealisation);
+                    }
+                }
         }
 
         StorePathSet closure;
         next->computeFSClosure(newPaths, closure);
         for (auto & path : closure)
             goal.addDependency(path);
+        for (auto & real : Realisation::closure(*next, newRealisations))
+            goal.addedDrvOutputs.insert(real.id);
     }
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index d30be2351..088a57209 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Paths that were added via recursive Nix calls. */
     StorePathSet addedPaths;
 
+    /* Realisations that were added via recursive Nix calls. */
+    std::set<DrvOutput> addedDrvOutputs;
+
     /* Recursive Nix calls are only allowed to build or realize paths
        in the original input closure or added via a recursive Nix call
        (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
+    bool isAllowed(const DrvOutput & id)
+    {
+        return addedDrvOutputs.count(id);
+    }
+
     bool isAllowed(const DerivedPath & req);
 
     friend struct RestrictedStore;
diff --git a/tests/ca/recursive.sh b/tests/ca/recursive.sh
new file mode 100755
index 000000000..d9281d91f
--- /dev/null
+++ b/tests/ca/recursive.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+cd ..
+source ./recursive.sh
+
+
diff --git a/tests/local.mk b/tests/local.mk
index 82cec1df3..4d9d314cc 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -52,6 +52,7 @@ nix_tests = \
   ca/signatures.sh \
   ca/nix-shell.sh \
   ca/nix-run.sh \
+  ca/recursive.sh \
   ca/nix-copy.sh
   # parallel.sh
 
diff --git a/tests/recursive.sh b/tests/recursive.sh
index a55b061b5..b6740877d 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -9,9 +9,9 @@ rm -f $TEST_ROOT/result
 
 export unreachable=$(nix store add-path ./recursive.sh)
 
-NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
+NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
   with import ./config.nix;
-  mkDerivation {
+  mkDerivation rec {
     name = "recursive";
     dummy = builtins.toFile "dummy" "bla bla";
     SHELL = shell;
@@ -19,11 +19,13 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r
     # Note: this is a string without context.
     unreachable = builtins.getEnv "unreachable";
 
+    NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT";
+
     requiredSystemFeatures = [ "recursive-nix" ];
 
     buildCommand = '\'\''
       mkdir $out
-      opts="--experimental-features nix-command"
+      opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}"
 
       PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH
 
@@ -46,16 +48,15 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r
       # Add it to our closure.
       ln -s $foobar $out/foobar
 
-      [[ $(nix $opts path-info --all | wc -l) -eq 3 ]]
+      [[ $(nix $opts path-info --all | wc -l) -eq 4 ]]
 
       # Build a derivation.
       nix $opts build -L --impure --expr '\''
-        derivation {
+        with import ${./config.nix};
+        mkDerivation {
           name = "inner1";
-          builder = builtins.getEnv "SHELL";
-          system = builtins.getEnv "system";
+          buildCommand = "echo $fnord blaat > $out";
           fnord = builtins.toFile "fnord" "fnord";
-          args = [ "-c" "echo $fnord blaat > $out" ];
         }
       '\''
 

From 8b6fba2b63820ea63599d200f79740bb7f85de1e Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 24 Jun 2021 15:44:13 +0200
Subject: [PATCH 46/90] Eventually delete the CA paths lock files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mark the lockfiles as having to eventually be deleted so that they don’t
stay laying around in the store at the end of the build

Fix #4936
---
 src/libstore/build/local-derivation-goal.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index ba0aca29c..8320dd1c4 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -2481,6 +2481,7 @@ void LocalDerivationGoal::registerOutputs()
            floating CA derivations and hash-mismatching fixed-output
            derivations. */
         PathLocks dynamicOutputLock;
+        dynamicOutputLock.setDeletion(true);
         auto optFixedPath = output.path(worker.store, drv->name, outputName);
         if (!optFixedPath ||
             worker.store.printStorePath(*optFixedPath) != finalDestPath)

From 57409244ec8f568ce6b4ecb1bde0e509942ac05f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 24 Jun 2021 18:02:51 +0200
Subject: [PATCH 47/90] boehmgc: Crude support for coroutines

Fixes the problem where a stack pointer outside the original
thread causes the collector to crash.

It could be made more accurate by recording the stack pointer
every time we switch to a coroutine. We can use this information
to update our own coroutine stacks like normal data. When the
stack pointer is on a thread, we can add a field to GC_thread
"fallback_sp" to be used when the thread sp is outside the original
thread range.
---
 boehmgc-coroutine-sp-fallback.diff | 44 ++++++++++++++++++++++++++++++
 flake.nix                          |  8 +++++-
 2 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 boehmgc-coroutine-sp-fallback.diff

diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff
new file mode 100644
index 000000000..c618718a7
--- /dev/null
+++ b/boehmgc-coroutine-sp-fallback.diff
@@ -0,0 +1,44 @@
+diff --git a/pthread_stop_world.c b/pthread_stop_world.c
+index 1cee6a0b..8977b0dd 100644
+--- a/pthread_stop_world.c
++++ b/pthread_stop_world.c
+@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
+     struct GC_traced_stack_sect_s *traced_stack_sect;
+     pthread_t self = pthread_self();
+     word total_size = 0;
++    size_t stack_limit;
++    pthread_attr_t pattr;
+ 
+     if (!EXPECT(GC_thr_initialized, TRUE))
+       GC_thr_init();
+@@ -723,6 +725,30 @@ GC_INNER void GC_push_all_stacks(void)
+           hi = p->altstack + p->altstack_size;
+           /* FIXME: Need to scan the normal stack too, but how ? */
+           /* FIXME: Assume stack grows down */
++        } else {
++          if (pthread_getattr_np(p->id, &pattr)) {
++            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
++          }
++          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
++            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
++          }
++          // When a thread goes into a coroutine, we lose its original sp until
++          // control flow returns to the thread.
++          // While in the coroutine, the sp points outside the thread stack,
++          // so we can detect this and push the entire thread stack instead,
++          // as an approximation.
++          // We assume that the coroutine has similarly added its entire stack.
++          // This could be made accurate by cooperating with the application
++          // via new functions and/or callbacks.
++          #ifndef STACK_GROWS_UP
++            if (lo >= hi || lo < hi - stack_limit) { // sp outside stack
++              lo = hi - stack_limit;
++            }
++          #else
++            if (lo < hi || lo >= hi + stack_limit) { // sp outside stack
++              lo = hi + stack_limit;
++            }
++          #endif
+         }
+         GC_push_all_stack_sections(lo, hi, traced_stack_sect);
+ #       ifdef STACK_GROWS_UP
diff --git a/flake.nix b/flake.nix
index 6c65e3406..8e7dd75d3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -102,7 +102,13 @@
           });
 
         propagatedDeps =
-          [ (boehmgc.override { enableLargeConfig = true; })
+          [ ((boehmgc.override {
+              enableLargeConfig = true;
+            }).overrideAttrs(o: {
+              patches = (o.patches or []) ++ [
+                ./boehmgc-coroutine-sp-fallback.diff
+              ];
+            }))
           ];
 
         perlDeps =

From bf68c693dc5157c2be1f3a9f407dd1ce3761df78 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Jun 2021 11:17:19 +0200
Subject: [PATCH 48/90] tests: Get rid of some result symlinks

Fixes

  error: cannot create symlink '/home/eelco/Dev/nix/tests/result'; already exists
---
 tests/build.sh                               | 6 +++---
 tests/ca/duplicate-realisation-in-closure.sh | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/build.sh b/tests/build.sh
index ce9d6602c..c77f620f7 100644
--- a/tests/build.sh
+++ b/tests/build.sh
@@ -1,7 +1,7 @@
 source common.sh
 
 expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
-nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
+nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
     (.outputs.first | match(".*multiple-outputs-a-first")) and
@@ -10,10 +10,10 @@ nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
     (.drvPath | match(".*multiple-outputs-b.drv")) and
     (.outputs.out | match(".*multiple-outputs-b")))
 '
-
 testNormalization () {
     clearStore
-    outPath=$(nix-build ./simple.nix)
+    outPath=$(nix-build ./simple.nix --no-out-link)
     test "$(stat -c %Y $outPath)" -eq 1
 }
+
 testNormalization
diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/ca/duplicate-realisation-in-closure.sh
index bfe2a4e08..ca9099641 100644
--- a/tests/ca/duplicate-realisation-in-closure.sh
+++ b/tests/ca/duplicate-realisation-in-closure.sh
@@ -17,10 +17,10 @@ sleep 2 # To make sure that `$(date)` will be different
 # As we’ve cleared the cache, we’ll have to rebuild current-time. And because
 # the current time isn’t the same as before, this will yield a new (different)
 # realisation
-nix build -f nondeterministic.nix dep2
+nix build -f nondeterministic.nix dep2 --no-link
 
 # Build something that depends both on dep1 and dep2.
 # If everything goes right, we should rebuild dep2 rather than fetch it from
 # the cache (because that would mean duplicating `current-time` in the closure),
 # and have `dep1 == dep2`.
-nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs
+nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link

From 5c58d84a76d96f269e3ff1e72c9c9ba5f68576af Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 25 Jun 2021 17:45:48 +0200
Subject: [PATCH 49/90] boehmgc: Remove unused code from patch

---
 boehmgc-coroutine-sp-fallback.diff | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff
index c618718a7..fa8dd0325 100644
--- a/boehmgc-coroutine-sp-fallback.diff
+++ b/boehmgc-coroutine-sp-fallback.diff
@@ -1,5 +1,5 @@
 diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index 1cee6a0b..8977b0dd 100644
+index 1cee6a0b..46c3acd9 100644
 --- a/pthread_stop_world.c
 +++ b/pthread_stop_world.c
 @@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
@@ -11,7 +11,7 @@ index 1cee6a0b..8977b0dd 100644
  
      if (!EXPECT(GC_thr_initialized, TRUE))
        GC_thr_init();
-@@ -723,6 +725,30 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void)
            hi = p->altstack + p->altstack_size;
            /* FIXME: Need to scan the normal stack too, but how ? */
            /* FIXME: Assume stack grows down */
@@ -35,9 +35,7 @@ index 1cee6a0b..8977b0dd 100644
 +              lo = hi - stack_limit;
 +            }
 +          #else
-+            if (lo < hi || lo >= hi + stack_limit) { // sp outside stack
-+              lo = hi + stack_limit;
-+            }
++          #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
 +          #endif
          }
          GC_push_all_stack_sections(lo, hi, traced_stack_sect);

From c906d6530dc52d79f69e65e0447893084a0d4556 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sat, 26 Jun 2021 00:12:03 -0500
Subject: [PATCH 50/90] Support cross-compiling binaryTarball

---
 flake.nix                     | 164 ++++++++++++++++++----------------
 maintainers/upload-release.pl |   2 +
 scripts/install.in            |  10 +++
 3 files changed, 99 insertions(+), 77 deletions(-)

diff --git a/flake.nix b/flake.nix
index 829cd2804..08b385ec2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -135,10 +135,11 @@
 
             substitute ${./scripts/install.in} $out/install \
               ${pkgs.lib.concatMapStrings
-                (system:
-                  '' \
-                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                  --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                (system: let
+                    tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system};
+                  in '' \
+                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
+                  --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
                   ''
                 )
                 systems
@@ -176,6 +177,77 @@
 
       };
 
+      binaryTarball = buildPackages: nix: pkgs: let
+        inherit (pkgs) cacert;
+        installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
+      in
+
+      buildPackages.runCommand "nix-binary-tarball-${version}"
+        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+          meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
+        }
+        ''
+          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+          substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+
+          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+
+          if type -p shellcheck; then
+            # SC1090: Don't worry about not being able to find
+            #         $nix/etc/profile.d/nix.sh
+            shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/create-darwin-volume.sh
+            shellcheck $TMPDIR/install-darwin-multi-user.sh
+            shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+            # SC1091: Don't panic about not being able to source
+            #         /etc/profile
+            # SC2002: Ignore "useless cat" "error", when loading
+            #         .reginfo, as the cat is a much cleaner
+            #         implementation, even though it is "useless"
+            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+            #         root's home directory
+            shellcheck --external-sources \
+              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+          fi
+
+          chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/create-darwin-volume.sh
+          chmod +x $TMPDIR/install-darwin-multi-user.sh
+          chmod +x $TMPDIR/install-systemd-multi-user.sh
+          chmod +x $TMPDIR/install-multi-user
+          dir=nix-${version}-${pkgs.system}
+          fn=$out/$dir.tar.xz
+          mkdir -p $out/nix-support
+          echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+          tar cvfJ $fn \
+            --owner=0 --group=0 --mode=u+rw,uga+r \
+            --absolute-names \
+            --hard-dereference \
+            --transform "s,$TMPDIR/install,$dir/install," \
+            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
+            --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+            --transform "s,$NIX_STORE,$dir/store,S" \
+            $TMPDIR/install \
+            $TMPDIR/create-darwin-volume.sh \
+            $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install-systemd-multi-user.sh \
+            $TMPDIR/install-multi-user \
+            $TMPDIR/reginfo \
+            $(cat ${installerClosureInfo}/store-paths)
+        '';
+
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
@@ -315,85 +387,23 @@
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
-        binaryTarball = nixpkgs.lib.genAttrs systems (system:
+        binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system});
 
-          with nixpkgsFor.${system};
-
-          let
-            installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
-          in
-
-          runCommand "nix-binary-tarball-${version}"
-            { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-              meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
-            }
-            ''
-              cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-              cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-              substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              if type -p shellcheck; then
-                # SC1090: Don't worry about not being able to find
-                #         $nix/etc/profile.d/nix.sh
-                shellcheck --exclude SC1090 $TMPDIR/install
-                shellcheck $TMPDIR/create-darwin-volume.sh
-                shellcheck $TMPDIR/install-darwin-multi-user.sh
-                shellcheck $TMPDIR/install-systemd-multi-user.sh
-
-                # SC1091: Don't panic about not being able to source
-                #         /etc/profile
-                # SC2002: Ignore "useless cat" "error", when loading
-                #         .reginfo, as the cat is a much cleaner
-                #         implementation, even though it is "useless"
-                # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-                #         root's home directory
-                shellcheck --external-sources \
-                  --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-              fi
-
-              chmod +x $TMPDIR/install
-              chmod +x $TMPDIR/create-darwin-volume.sh
-              chmod +x $TMPDIR/install-darwin-multi-user.sh
-              chmod +x $TMPDIR/install-systemd-multi-user.sh
-              chmod +x $TMPDIR/install-multi-user
-              dir=nix-${version}-${system}
-              fn=$out/$dir.tar.xz
-              mkdir -p $out/nix-support
-              echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-              tar cvfJ $fn \
-                --owner=0 --group=0 --mode=u+rw,uga+r \
-                --absolute-names \
-                --hard-dereference \
-                --transform "s,$TMPDIR/install,$dir/install," \
-                --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
-                --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-                --transform "s,$NIX_STORE,$dir/store,S" \
-                $TMPDIR/install \
-                $TMPDIR/create-darwin-volume.sh \
-                $TMPDIR/install-darwin-multi-user.sh \
-                $TMPDIR/install-systemd-multi-user.sh \
-                $TMPDIR/install-multi-user \
-                $TMPDIR/reginfo \
-                $(cat ${installerClosureInfo}/store-paths)
-            '');
+        binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: {
+          name = crossSystem;
+          value = let
+            nixpkgsCross = import nixpkgs {
+              inherit system crossSystem;
+              overlays = [ self.overlay ];
+            };
+          in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross;
+        }) crossSystems));
 
         # The first half of the installation script. This is uploaded
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
+        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
         installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ];
 
         # Line coverage analysis.
diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index c2933300f..77007f914 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -110,6 +110,8 @@ downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
 downloadFile("installerScript", "1");
 
 for my $fn (glob "$tmpDir/*") {
diff --git a/scripts/install.in b/scripts/install.in
index 39016d161..e801d4268 100755
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -40,6 +40,16 @@ case "$(uname -s).$(uname -m)" in
         path=@tarballPath_aarch64-linux@
         system=aarch64-linux
         ;;
+    Linux.armv6l_linux)
+        hash=@tarballHash_armv6l-linux@
+        path=@tarballPath_armv6l-linux@
+        system=armv6l-linux
+        ;;
+    Linux.armv7l_linux)
+        hash=@tarballHash_armv7l-linux@
+        path=@tarballPath_armv7l-linux@
+        system=armv7l-linux
+        ;;
     Darwin.x86_64)
         hash=@tarballHash_x86_64-darwin@
         path=@tarballPath_x86_64-darwin@

From 580583e0b386df181a3c30519c821275362b10e0 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sat, 26 Jun 2021 00:14:54 -0500
Subject: [PATCH 51/90] Build cross-compilation in gha

---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 08b385ec2..8b37a5658 100644
--- a/flake.nix
+++ b/flake.nix
@@ -404,7 +404,7 @@
         # tarball for the user's system and calls the second half of the
         # installation script.
         installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ];
+        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
         # Line coverage analysis.
         coverage =

From decc14d4b7fbe1664ae609563003c883d4e426a8 Mon Sep 17 00:00:00 2001
From: Florian Franzen <Florian.Franzen@gmail.com>
Date: Mon, 28 Jun 2021 16:27:03 +0200
Subject: [PATCH 52/90] man: fix formatting of nix3-profile-remove

---
 src/nix/profile-remove.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md
index dcf825da9..ba85441d8 100644
--- a/src/nix/profile-remove.md
+++ b/src/nix/profile-remove.md
@@ -15,6 +15,7 @@ R""(
   ```
 
 * Remove all packages:
+
   ```console
   # nix profile remove '.*'
   ```

From 777c688a9875c1b1864246b28b128f5431cbc6d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Mon, 28 Jun 2021 18:42:44 +0200
Subject: [PATCH 53/90] flake.lock: Update

Flake input changes:

* Updated 'nixpkgs': 'github:NixOS/nixpkgs/bb8a5e54845012ed1375ffd5f317d2fdf434b20e' -> 'github:NixOS/nixpkgs/f77036342e2b690c61c97202bf48f2ce13acc022'
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 8aad22957..5fc969d7b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -19,11 +19,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1622593737,
-        "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=",
+        "lastModified": 1624862269,
+        "narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e",
+        "rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
         "type": "github"
       },
       "original": {

From 6c13a3f7354d02fb3a38604ed601b76c011e826f Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Mon, 28 Jun 2021 15:08:17 -0500
Subject: [PATCH 54/90] Support binaryTarballCross in gha

---
 flake.nix | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 8b37a5658..c97130ac9 100644
--- a/flake.nix
+++ b/flake.nix
@@ -387,7 +387,7 @@
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
-        binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system});
+        binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system});
 
         binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: {
           name = crossSystem;
@@ -498,7 +498,10 @@
             # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
             # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
           } "touch $out";
-      });
+      } // (if system == "x86_64-linux" then (builtins.listToAttrs (map (crossSystem: {
+        name = "binaryTarball-${crossSystem}";
+        value = self.hydraJobs.binaryTarballCross.${system}.${crossSystem};
+      }) crossSystems)) else {}));
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;

From 9feca5cdf64b82bfb06dfda07d19d007a2dfa1c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Mon, 28 Jun 2021 23:02:53 +0200
Subject: [PATCH 55/90] github actions: simplify getting the system logic

---
 .github/workflows/test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 17a79dc97..b2b1f07fb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,8 +20,7 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    #- run: nix flake check
-    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
+    - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest

From bf7960a4ede0f5ba7e88b939ed0f33aa6087cbc6 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 29 Jun 2021 10:13:42 +0200
Subject: [PATCH 56/90] develop: Discard the input{Overrides,Updates} when
 getting bash
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`nix develop` is getting bash from an (assumed existing) `nixpkgs`
flake. However, when doing so, it reuses the `lockFlags` passed to the
current flake, including the `--input-overrides` and `--input-update`
which generally don’t make sense anymore at that point (and trigger a
warning because of that)

Clear these overrides before getting the nixpkgs flake to get rid of the
warning.
---
 src/nix/develop.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index d77ff52d7..6c089469d 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -443,13 +443,17 @@ struct CmdDevelop : Common, MixEnvironment
         try {
             auto state = getEvalState();
 
+            auto nixpkgsLockFlags = lockFlags;
+            nixpkgsLockFlags.inputOverrides = {};
+            nixpkgsLockFlags.inputUpdates = {};
+
             auto bashInstallable = std::make_shared<InstallableFlake>(
                 this,
                 state,
                 installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
-                lockFlags);
+                nixpkgsLockFlags);
 
             shell = state->store->printStorePath(
                 toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash";

From 4cff8188a5b59303f726a2497727fff34f9711d3 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 29 Jun 2021 13:53:56 +0200
Subject: [PATCH 57/90] Keep the `isFlake` attribute for overriden inputs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When overriding an input that isn’t a flake, mark the override as also not
being a flake.

Fix #3774
---
 src/libexpr/flake/flake.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 2e94490d4..8e6f06949 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -359,7 +359,12 @@ LockedFlake lockFlake(
                        ancestors? */
                     auto i = overrides.find(inputPath);
                     bool hasOverride = i != overrides.end();
-                    if (hasOverride) overridesUsed.insert(inputPath);
+                    if (hasOverride) {
+                        overridesUsed.insert(inputPath);
+                        // Respect the “flakeness” of the input even if we
+                        // override it
+                        i->second.isFlake = input2.isFlake;
+                    }
                     auto & input = hasOverride ? i->second : input2;
 
                     /* Resolve 'follows' later (since it may refer to an input

From 7daf0c6ef12a0fbeedb53ed683c93478764fe1ab Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 29 Jun 2021 14:52:46 +0200
Subject: [PATCH 58/90] Forward the experimental features to the nix repl
 subprocesses
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pass the current experimental features using `NIX_CONFIG` to the various
Nix subprocesses that `nix repl` invokes.

This is quite a hack, but having `nix repl` call Nix with a subprocess
is a hack already, so I guess that’s fine.
---
 src/nix/repl.cc | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index eed79c332..0275feae7 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -104,6 +104,26 @@ NixRepl::~NixRepl()
     write_history(historyFile.c_str());
 }
 
+string runNix(Path program, const Strings & args,
+    const std::optional<std::string> & input = {})
+{
+    auto experimentalFeatures = concatStringsSep(" ", settings.experimentalFeatures.get());
+    auto nixConf = getEnv("NIX_CONFIG").value_or("");
+    nixConf.append("\nexperimental-features = " + experimentalFeatures);
+    auto subprocessEnv = getEnv();
+    subprocessEnv["NIX_CONFIG"] = nixConf;
+    RunOptions opts(settings.nixBinDir+ "/" + program, args);
+    opts.input = input;
+    opts.environment = subprocessEnv;
+
+    auto res = runProgram(opts);
+
+    if (!statusOk(res.first))
+        throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
+
+    return res.second;
+}
+
 static NixRepl * curRepl; // ugly
 
 static char * completionCallback(char * s, int *match) {
@@ -463,7 +483,7 @@ bool NixRepl::processLine(string line)
         state->callFunction(f, v, result, Pos());
 
         StorePath drvPath = getDerivationPath(result);
-        runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)});
+        runNix("nix-shell", {state->store->printStorePath(drvPath)});
     }
 
     else if (command == ":b" || command == ":i" || command == ":s") {
@@ -477,7 +497,7 @@ bool NixRepl::processLine(string line)
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
             try {
-                runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw});
+                runNix("nix", {"build", "--no-link", drvPathRaw});
                 auto drv = state->store->readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputsAndOptPaths(*state->store))
@@ -485,9 +505,9 @@ bool NixRepl::processLine(string line)
             } catch (ExecError &) {
             }
         } else if (command == ":i") {
-            runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw});
+            runNix("nix-env", {"-i", drvPathRaw});
         } else {
-            runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw});
+            runNix("nix-shell", {drvPathRaw});
         }
     }
 

From 7351656b82e59e46610c7e9edee023615b153e0e Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 29 Jun 2021 21:46:54 -0500
Subject: [PATCH 59/90] Only cross compile from x86_64-linux

This is broken on aarch64-linux / x86_64-darwin, so might as well just disable it for now.
---
 flake.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 999028ec2..7d41947a7 100644
--- a/flake.nix
+++ b/flake.nix
@@ -385,7 +385,7 @@
         buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
 
         buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem:
-          nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}."nix-${crossSystem}"));
+          nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
 
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
@@ -395,7 +395,7 @@
         # the installation script.
         binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system});
 
-        binaryTarballCross = nixpkgs.lib.genAttrs systems (system: builtins.listToAttrs (map (crossSystem: {
+        binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: {
           name = crossSystem;
           value = let
             nixpkgsCross = import nixpkgs {

From 2200f315daff989a974ed4fac445f90a06d55e81 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 29 Jun 2021 21:48:07 -0500
Subject: [PATCH 60/90] Disable -pie on static nix

This should resolve the failing build.

See https://github.com/NixOS/nixpkgs/pull/128674 for a better fix.
---
 flake.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flake.nix b/flake.nix
index 7d41947a7..5430d3904 100644
--- a/flake.nix
+++ b/flake.nix
@@ -549,6 +549,8 @@
           stripAllList = ["bin"];
 
           strictDeps = true;
+
+          hardeningDisable = [ "pie" ];
         };
       } // builtins.listToAttrs (map (crossSystem: {
         name = "nix-${crossSystem}";

From 5be17a4b96fa4dbd512da4cba59f177cdbd84935 Mon Sep 17 00:00:00 2001
From: Pamplemousse <xav.maso@gmail.com>
Date: Fri, 25 Jun 2021 15:57:37 -0700
Subject: [PATCH 61/90] Allow to compile after `./configure --enable-gc=no`

Signed-off-by: Pamplemousse <xav.maso@gmail.com>
---
 src/libexpr/eval.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index ef9f8efca..ceca6e16b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -64,7 +64,11 @@ static char * dupStringWithLen(const char * s, size_t size)
 
 RootValue allocRootValue(Value * v)
 {
+#if HAVE_BOEHMGC
     return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
+#else
+    return std::make_shared<Value *>(v);
+#endif
 }
 
 

From 093ed4763615a73edb1e8567d2f41d55ff5aaa0c Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Wed, 30 Jun 2021 22:13:32 +0300
Subject: [PATCH 62/90] nix registry: add --registry flag

---
 src/nix/registry-add.md    |  7 ++++
 src/nix/registry-pin.md    |  7 ++++
 src/nix/registry-remove.md |  6 ++++
 src/nix/registry.cc        | 67 ++++++++++++++++++++++++++++++--------
 4 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md
index 80a31996a..a947fa0b3 100644
--- a/src/nix/registry-add.md
+++ b/src/nix/registry-add.md
@@ -21,6 +21,13 @@ R""(
   # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs
   ```
 
+* Add `nixpkgs` pointing to `github:nixos/nixpkgs` to your custom flake
+  registry:
+
+  ```console
+  nix registry add --registry ./custom-flake-registry.json nixpkgs github:nixos/nixpkgs
+  ```
+
 # Description
 
 This command adds an entry to the user registry that maps flake
diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md
index 6e97e003e..7b163c463 100644
--- a/src/nix/registry-pin.md
+++ b/src/nix/registry-pin.md
@@ -24,6 +24,13 @@ R""(
   …
   ```
 
+* Pin `nixpkgs` in a custom registry to its most recent Git revision
+
+  ```console
+  # nix registry pin --registry ./custom-flake-registry.json nixpkgs
+  ```
+
+
 # Description
 
 This command adds an entry to the user registry that maps flake
diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md
index 4c0eb4947..eecd4c6e7 100644
--- a/src/nix/registry-remove.md
+++ b/src/nix/registry-remove.md
@@ -8,6 +8,12 @@ R""(
   # nix registry remove nixpkgs
   ```
 
+* Remove the entry `nixpkgs` from a custom registry:
+
+  ```console
+  # nix registry remove --registry ./custom-flake-registry.json nixpkgs
+  ```
+
 # Description
 
 This command removes from the user registry any entry for flake
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index f9719600f..4400cc8dc 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -10,6 +10,45 @@
 using namespace nix;
 using namespace nix::flake;
 
+
+class RegistryCommand: virtual Args
+{
+    std::string registry_path;
+
+    std::shared_ptr<fetchers::Registry> registry;
+
+
+public:
+
+    RegistryCommand()
+    {
+        addFlag({
+            .longName = "registry",
+            .description = "The registry to operate on.",
+            .labels = {"registry"},
+            .handler = {&registry_path},
+        });
+    }
+
+    std::shared_ptr<fetchers::Registry> getRegistry() {
+        if (registry) return registry;
+        if (registry_path.empty()) {
+            registry = fetchers::getUserRegistry();
+        } else {
+            registry = fetchers::getCustomRegistry(registry_path);
+        }
+        return registry;
+    }
+
+    Path getRegistryPath() {
+        if (registry_path.empty()) {
+            return fetchers::getUserRegistryPath();
+        } else {
+            return registry_path;
+        }
+    }
+};
+
 struct CmdRegistryList : StoreCommand
 {
     std::string description() override
@@ -45,7 +84,7 @@ struct CmdRegistryList : StoreCommand
     }
 };
 
-struct CmdRegistryAdd : MixEvalArgs, Command
+struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand
 {
     std::string fromUrl, toUrl;
 
@@ -71,16 +110,16 @@ struct CmdRegistryAdd : MixEvalArgs, Command
     {
         auto fromRef = parseFlakeRef(fromUrl);
         auto toRef = parseFlakeRef(toUrl);
+        auto registry = getRegistry();
         fetchers::Attrs extraAttrs;
         if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir;
-        auto userRegistry = fetchers::getUserRegistry();
-        userRegistry->remove(fromRef.input);
-        userRegistry->add(fromRef.input, toRef.input, extraAttrs);
-        userRegistry->write(fetchers::getUserRegistryPath());
+        registry->remove(fromRef.input);
+        registry->add(fromRef.input, toRef.input, extraAttrs);
+        registry->write(getRegistryPath());
     }
 };
 
-struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command
+struct CmdRegistryRemove : RegistryCommand, Command
 {
     std::string url;
 
@@ -103,13 +142,13 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command
 
     void run() override
     {
-        auto userRegistry = fetchers::getUserRegistry();
-        userRegistry->remove(parseFlakeRef(url).input);
-        userRegistry->write(fetchers::getUserRegistryPath());
+        auto registry = getRegistry();
+        registry->remove(parseFlakeRef(url).input);
+        registry->write(getRegistryPath());
     }
 };
 
-struct CmdRegistryPin : virtual Args, EvalCommand
+struct CmdRegistryPin : RegistryCommand, EvalCommand
 {
     std::string url;
 
@@ -132,14 +171,14 @@ struct CmdRegistryPin : virtual Args, EvalCommand
 
     void run(nix::ref<nix::Store> store) override
     {
+        auto registry = getRegistry();
         auto ref = parseFlakeRef(url);
-        auto userRegistry = fetchers::getUserRegistry();
-        userRegistry->remove(ref.input);
+        registry->remove(ref.input);
         auto [tree, resolved] = ref.resolve(store).input.fetch(store);
         fetchers::Attrs extraAttrs;
         if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
-        userRegistry->add(ref.input, resolved, extraAttrs);
-        userRegistry->write(fetchers::getUserRegistryPath());
+        registry->add(ref.input, resolved, extraAttrs);
+        registry->write(getRegistryPath());
     }
 };
 

From 811f3e8605ce779830fa4770cd7ef3cda81afdf1 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Wed, 30 Jun 2021 22:14:41 +0300
Subject: [PATCH 63/90] nix registry pin: add a way to pin to a custom locked

---
 src/libfetchers/registry.cc |  7 +++++++
 src/libfetchers/registry.hh |  3 +++
 src/nix/registry.cc         | 19 +++++++++++++++++--
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 74376adc0..c2dfa9dc6 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry()
     return userRegistry;
 }
 
+std::shared_ptr<Registry> getCustomRegistry(Path p)
+{
+    static auto customRegistry =
+        Registry::read(p, Registry::Custom);
+    return customRegistry;
+}
+
 static std::shared_ptr<Registry> flagRegistry =
     std::make_shared<Registry>(Registry::Flag);
 
diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh
index 1077af020..cde22dff6 100644
--- a/src/libfetchers/registry.hh
+++ b/src/libfetchers/registry.hh
@@ -14,6 +14,7 @@ struct Registry
         User = 1,
         System = 2,
         Global = 3,
+        Custom = 4,
     };
 
     RegistryType type;
@@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
 
 std::shared_ptr<Registry> getUserRegistry();
 
+std::shared_ptr<Registry> getCustomRegistry(Path p);
+
 Path getUserRegistryPath();
 
 Registries getRegistries(ref<Store> store);
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index 4400cc8dc..ec80dc0bc 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -152,9 +152,11 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand
 {
     std::string url;
 
+    std::string locked;
+
     std::string description() override
     {
-        return "pin a flake to its current version in user flake registry";
+        return "pin a flake to its current version in user flake registry or to the current version of a flake URI";
     }
 
     std::string doc() override
@@ -167,14 +169,27 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand
     CmdRegistryPin()
     {
         expectArg("url", &url);
+
+        expectArgs({
+            .label = "locked",
+            .optional = true,
+            .handler = {&locked},
+            .completer = {[&](size_t, std::string_view prefix) {
+                completeFlakeRef(getStore(), prefix);
+            }}
+        });
     }
 
     void run(nix::ref<nix::Store> store) override
     {
+        if (locked.empty()) {
+            locked = url;
+        }
         auto registry = getRegistry();
         auto ref = parseFlakeRef(url);
+        auto locked_ref = parseFlakeRef(locked);
         registry->remove(ref.input);
-        auto [tree, resolved] = ref.resolve(store).input.fetch(store);
+        auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store);
         fetchers::Attrs extraAttrs;
         if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
         registry->add(ref.input, resolved, extraAttrs);

From ef1e7ab840eaa50da29ddde7bf990a1ff6a8b185 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Thu, 1 Jul 2021 00:23:47 +0300
Subject: [PATCH 64/90] flake.nixConfig: fix flake-registry config settings

Before this commit, nixConfig.flake-registry didn't have any real effect
on the evaluation, since config was applied after inputs were evaluated.
Change this behavior: apply the config in the beginning of flake::lockFile.
---
 src/libcmd/installables.cc | 2 --
 src/libexpr/flake/flake.cc | 3 +++
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index fe52912cf..540e53eee 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -575,8 +575,6 @@ std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
 {
     if (!_lockedFlake) {
         _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
-        _lockedFlake->flake.config.apply();
-        // FIXME: send new config to the daemon.
     }
     return _lockedFlake;
 }
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 8e6f06949..b57679d3b 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -298,6 +298,9 @@ LockedFlake lockFlake(
 
     auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
 
+    flake.config.apply();
+    // FIXME: send new config to the daemon.
+
     try {
 
         // FIXME: symlink attack

From e756a59c72e3e793ed01dec04749fa356f263315 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Thu, 1 Jul 2021 17:54:22 +0300
Subject: [PATCH 65/90] fixup! flake.nixConfig: fix flake-registry config
 settings

---
 src/libcmd/installables.cc | 4 +++-
 src/libexpr/flake/flake.cc | 6 ++++--
 src/libexpr/flake/flake.hh | 4 ++++
 src/nix/flake.cc           | 4 ++++
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 540e53eee..269be11a2 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -573,8 +573,10 @@ InstallableFlake::getCursors(EvalState & state)
 
 std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
 {
+    flake::LockFlags lockFlagsApplyConfig = lockFlags;
+    lockFlagsApplyConfig.applyNixConfig = true;
     if (!_lockedFlake) {
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index b57679d3b..e266bc36d 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -298,8 +298,10 @@ LockedFlake lockFlake(
 
     auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
 
-    flake.config.apply();
-    // FIXME: send new config to the daemon.
+    if (lockFlags.applyNixConfig) {
+        flake.config.apply();
+        // FIXME: send new config to the daemon.
+    }
 
     try {
 
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index d17d5e183..4479e95db 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -104,6 +104,10 @@ struct LockFlags
        references like 'nixpkgs'. */
     bool useRegistries = true;
 
+    /* Whether to apply flake's nixConfig attribute to the configuration */
+
+    bool applyNixConfig = false;
+
     /* Whether mutable flake references (i.e. those without a Git
        revision or similar) without a corresponding lock are
        allowed. Mutable flake references with a lock are always
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 64fcfc000..9055b07eb 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -84,6 +84,7 @@ struct CmdFlakeUpdate : FlakeCommand
 
         lockFlags.recreateLockFile = true;
         lockFlags.writeLockFile = true;
+        lockFlags.applyNixConfig = true;
 
         lockFlake();
     }
@@ -114,6 +115,7 @@ struct CmdFlakeLock : FlakeCommand
         settings.tarballTtl = 0;
 
         lockFlags.writeLockFile = true;
+        lockFlags.applyNixConfig = true;
 
         lockFlake();
     }
@@ -270,6 +272,8 @@ struct CmdFlakeCheck : FlakeCommand
         settings.readOnlyMode = !build;
 
         auto state = getEvalState();
+
+        lockFlags.applyNixConfig = true;
         auto flake = lockFlake();
 
         bool hasErrors = false;

From 4a7a8b87cd0dd383f1059fe7903c4b001314f58d Mon Sep 17 00:00:00 2001
From: Pamplemousse <xav.maso@gmail.com>
Date: Tue, 29 Jun 2021 14:28:43 -0700
Subject: [PATCH 66/90] Prefer to throw specific errors

Signed-off-by: Pamplemousse <xav.maso@gmail.com>
---
 src/libcmd/command.cc    | 2 +-
 src/libexpr/attr-path.cc | 6 +++---
 src/libexpr/eval.cc      | 2 +-
 src/libutil/url.cc       | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 569c4b9e4..6777b23be 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -188,7 +188,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
     }
 
     if (result.size() != 1)
-        throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
+        throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
     updateProfile(result[0]);
 }
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 9dd557205..867eb13a5 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error("missing closing quote in selection path '%1%'", s);
+                    throw ParseError("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -116,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     auto colon = pos.rfind(':');
     if (colon == std::string::npos)
-        throw Error("cannot parse meta.position attribute '%s'", pos);
+        throw ParseError("cannot parse meta.position attribute '%s'", pos);
 
     std::string filename(pos, 0, colon);
     unsigned int lineno;
     try {
         lineno = std::stoi(std::string(pos, colon + 1));
     } catch (std::invalid_argument & e) {
-        throw Error("cannot parse line number '%s'", pos);
+        throw ParseError("cannot parse line number '%s'", pos);
     }
 
     Symbol file = state.symbols.create(filename);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index ef9f8efca..657e4e4cf 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -908,7 +908,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
         // computation.
         if (mustBeTrivial &&
             !(dynamic_cast<ExprAttrs *>(e)))
-            throw Error("file '%s' must be an attribute set", path);
+            throw EvalError("file '%s' must be an attribute set", path);
         eval(e, v);
     } catch (Error & e) {
         addErrorTrace(e, "while evaluating the file '%1%':", path2);
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index c1bab866c..f6232d255 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url)
         auto isFile = scheme.find("file") != std::string::npos;
 
         if (authority && *authority != "" && isFile)
-            throw Error("file:// URL '%s' has unexpected authority '%s'",
+            throw BadURL("file:// URL '%s' has unexpected authority '%s'",
                 url, *authority);
 
         if (isFile && path.empty())

From 20cce079f258dc3f62340ecd7b9de1bbd99ef814 Mon Sep 17 00:00:00 2001
From: "Yestin L. Harrison" <yestin@ylh.io>
Date: Thu, 1 Jul 2021 18:19:01 -0600
Subject: [PATCH 67/90] Respect TERM=dumb more consistently

---
 src/libmain/progress-bar.cc | 2 +-
 src/libutil/logging.cc      | 2 +-
 src/libutil/util.cc         | 4 ++++
 src/libutil/util.hh         | 3 +++
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 15354549a..b2a6e2a82 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -484,7 +484,7 @@ Logger * makeProgressBar(bool printBuildLogs)
 {
     return new ProgressBar(
         printBuildLogs,
-        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+        shouldANSI()
     );
 }
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index d2e801175..6b9b850ca 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -46,7 +46,7 @@ public:
         : printBuildLogs(printBuildLogs)
     {
         systemd = getEnv("IN_SYSTEMD") == "1";
-        tty = isatty(STDERR_FILENO);
+        tty = shouldANSI();
     }
 
     bool isVerbose() override {
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 7e57fd7ca..94460b367 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1372,6 +1372,10 @@ void ignoreException()
     }
 }
 
+bool shouldANSI()
+{
+	return isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb";
+}
 
 std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index f84d0fb31..a8dd4bd47 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -482,6 +482,9 @@ constexpr char treeLast[] = "└───";
 constexpr char treeLine[] = "│   ";
 constexpr char treeNull[] = "    ";
 
+/* Determine whether ANSI escape sequences are appropriate for the
+   present output. */
+bool shouldANSI();
 
 /* Truncate a string to 'width' printable characters. If 'filterAll'
    is true, all ANSI escape sequences are filtered out. Otherwise,

From 74838deeb8b9c92fb9aa8abfb53d3f7f31db5f22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Fri, 2 Jul 2021 14:12:00 +0200
Subject: [PATCH 68/90] upload-release.pl: add aarch64-darwin

---
 maintainers/upload-release.pl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 77007f914..3a8d27499 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -110,6 +110,7 @@ downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("binaryTarball.aarch64-darwin", "1");
 downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
 downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
 downloadFile("installerScript", "1");
@@ -155,6 +156,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
            "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
            "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
            "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+           "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
            "}\n");
 
 system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;

From d8ad6f1c108ec253b307c30aadee460cf602dbd1 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Fri, 2 Jul 2021 16:10:57 +0300
Subject: [PATCH 69/90] Add tests for --registry and second arg for nix
 registry pin

---
 tests/flakes.sh | 80 +++++++------------------------------------------
 1 file changed, 10 insertions(+), 70 deletions(-)

diff --git a/tests/flakes.sh b/tests/flakes.sh
index 9764e1a6c..a4e657980 100644
--- a/tests/flakes.sh
+++ b/tests/flakes.sh
@@ -90,76 +90,14 @@ EOF
 git -C $nonFlakeDir add README.md
 git -C $nonFlakeDir commit -m 'Initial'
 
-cat > $registry <<EOF
-{
-  "version": 2,
-  "flakes": [
-    { "from": {
-        "type": "indirect",
-        "id": "flake1"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$flake1Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake2"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$flake2Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake3"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$flake3Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake4"
-      },
-      "to": {
-        "type": "indirect",
-        "id": "flake3"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake5"
-      },
-      "to": {
-        "type": "hg",
-        "url": "file://$flake5Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "nixpkgs"
-      },
-      "to": {
-        "type": "indirect",
-        "id": "flake1"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "templates"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$templatesDir"
-      }
-    }
-  ]
-}
-EOF
+# Construct a custom registry, additionally test the --registry flag
+nix registry add --registry $registry flake1 git+file://$flake1Dir
+nix registry add --registry $registry flake2 git+file://$flake2Dir
+nix registry add --registry $registry flake3 git+file://$flake3Dir
+nix registry add --registry $registry flake4 flake3
+nix registry add --registry $registry flake5 hg+file://$flake5Dir
+nix registry add --registry $registry nixpkgs flake1
+nix registry add --registry $registry templates git+file://$templatesDir
 
 # Test 'nix flake list'.
 [[ $(nix registry list | wc -l) == 7 ]]
@@ -405,6 +343,8 @@ nix registry add flake1 flake3
 [[ $(nix registry list | wc -l) == 8 ]]
 nix registry pin flake1
 [[ $(nix registry list | wc -l) == 8 ]]
+nix registry pin flake1 flake3
+[[ $(nix registry list | wc -l) == 8 ]]
 nix registry remove flake1
 [[ $(nix registry list | wc -l) == 7 ]]
 

From 0fe84bef7286bf9b38dbb628f450ac07079f8fed Mon Sep 17 00:00:00 2001
From: "Yestin L. Harrison" <yestin@ylh.io>
Date: Fri, 2 Jul 2021 09:33:54 -0600
Subject: [PATCH 70/90] Add $NO_COLOR check to ANSI escape conditions

---
 src/libutil/util.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 94460b367..ee9f17228 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1374,7 +1374,9 @@ void ignoreException()
 
 bool shouldANSI()
 {
-	return isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb";
+    return isatty(STDERR_FILENO)
+        && getEnv("TERM").value_or("dumb") != "dumb"
+        && !getEnv("NO_COLOR").has_value();
 }
 
 std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)

From ec2c6bd47041bda1f30e66ee48de662ddcd07378 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 2 Jul 2021 19:20:07 +0200
Subject: [PATCH 71/90] Allow scp-style uris in `fetchgit`

Fix #5303
---
 src/libexpr/primops/fetchTree.cc | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index b8b99d4fa..c593400a7 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -7,6 +7,7 @@
 
 #include <ctime>
 #include <iomanip>
+#include <regex>
 
 namespace nix {
 
@@ -60,10 +61,19 @@ void emitTreeAttrs(
     v.attrs->sort();
 }
 
-std::string fixURI(std::string uri, EvalState &state)
+std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
 {
     state.checkURI(uri);
-    return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+    return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
+}
+
+std::string fixURIForGit(std::string uri, EvalState & state)
+{
+    static std::regex scp_uri("([^/].*)@(.*):(.*)");
+    if (uri[0] != '/' && std::regex_match(uri, scp_uri))
+        return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
+    else
+        return fixURI(uri, state);
 }
 
 void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
@@ -121,15 +131,15 @@ static void fetchTree(
 
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
-        auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
+        auto url = state.coerceToString(pos, *args[0], context, false, false);
 
         if (type == "git") {
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
-            attrs.emplace("url", url);
+            attrs.emplace("url", fixURIForGit(url, state));
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
-            input = fetchers::Input::fromURL(url);
+            input = fetchers::Input::fromURL(fixURI(url, state));
         }
     }
 

From 70cb2ffaccb91eff6f4afe2552d0784e279f1fe9 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 3 Jul 2021 14:19:10 +0200
Subject: [PATCH 72/90] libcmd/installables: implement completion for Nix
 expressions passed via `-f`

---
 src/libcmd/installables.cc | 44 ++++++++++++++++++++++++++++++++------
 1 file changed, 37 insertions(+), 7 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index fe52912cf..49f063334 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -171,14 +171,44 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 
 void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
-    if (file) return; // FIXME
+    if (file) {
+        evalSettings.pureEval = false;
+        auto state = getEvalState();
+        Expr *e = state->parseExprFromFile(
+            resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
+        );
 
-    completeFlakeRefWithFragment(
-        getEvalState(),
-        lockFlags,
-        getDefaultFlakeAttrPathPrefixes(),
-        getDefaultFlakeAttrPaths(),
-        prefix);
+        Value root;
+        state->eval(e, root);
+
+        auto autoArgs = getAutoArgs(*state);
+
+        std::string prefix_ = std::string(prefix);
+        auto sep = prefix_.rfind('.');
+        if (sep != std::string::npos) {
+            prefix_.erase(sep);
+        } else {
+            prefix_ = "";
+        }
+
+        Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first);
+        state->forceValue(v1);
+        Value v2;
+        state->autoCallFunction(*autoArgs, v1, v2);
+
+        if (v2.type() == nAttrs) {
+            for (auto & i : *v2.attrs) {
+                completions->add(i.name);
+            }
+        }
+    } else {
+        completeFlakeRefWithFragment(
+            getEvalState(),
+            lockFlags,
+            getDefaultFlakeAttrPathPrefixes(),
+            getDefaultFlakeAttrPaths(),
+            prefix);
+    }
 }
 
 void completeFlakeRefWithFragment(

From 087c5f5325c46485a9bc5f8e7f2620af6bf6bf56 Mon Sep 17 00:00:00 2001
From: Michael Fellinger <michael.fellinger@iohk.io>
Date: Mon, 5 Jul 2021 12:00:08 +0200
Subject: [PATCH 73/90] Fix devShell handling of env values including @ and %

---
 src/nix/develop.cc | 2 +-
 tests/shell.nix    | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 6c089469d..699ec0b99 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -54,7 +54,7 @@ BuildEnvironment readEnvironment(const Path & path)
         R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
 
     static std::string simpleStringRegex =
-        R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
+        R"re((?:[a-zA-Z0-9_/:\.\-\+=@%]*))re";
 
     static std::string dquotedStringRegex =
         R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
diff --git a/tests/shell.nix b/tests/shell.nix
index 53759f99a..f174db583 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -34,6 +34,8 @@ let pkgs = rec {
     name = "shellDrv";
     builder = "/does/not/exist";
     VAR_FROM_NIX = "bar";
+    ASCII_PERCENT = "%";
+    ASCII_AT = "@";
     TEST_inNixShell = if inNixShell then "true" else "false";
     inherit stdenv;
     outputs = ["dev" "out"];

From c8a80e4dbe9c743ac0280a2504a0d451e8848b28 Mon Sep 17 00:00:00 2001
From: Jarrett Keifer <jkeifer@users.noreply.github.com>
Date: Mon, 5 Jul 2021 07:48:53 -0700
Subject: [PATCH 74/90] Fix wrong hash var for aarch64-darwin binary

---
 scripts/install.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install.in b/scripts/install.in
index e801d4268..ffc1f2785 100755
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -56,7 +56,7 @@ case "$(uname -s).$(uname -m)" in
         system=x86_64-darwin
         ;;
     Darwin.arm64|Darwin.aarch64)
-        hash=@binaryTarball_aarch64-darwin@
+        hash=@tarballHash_aarch64-darwin@
         path=@tarballPath_aarch64-darwin@
         system=aarch64-darwin
         ;;

From 3c5f69bb60f858f471121adaf172edb47628188e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 5 Jul 2021 21:37:33 +0200
Subject: [PATCH 75/90] completeInstallable: also match for already typed
 prefixes

---
 src/libcmd/installables.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 49f063334..5f263061b 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -185,9 +185,12 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
 
         std::string prefix_ = std::string(prefix);
         auto sep = prefix_.rfind('.');
+        std::string searchWord;
         if (sep != std::string::npos) {
-            prefix_.erase(sep);
+            searchWord = prefix_.substr(sep, std::string::npos);
+            prefix_ = prefix_.substr(0, sep);
         } else {
+            searchWord = prefix_;
             prefix_ = "";
         }
 
@@ -198,7 +201,10 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
 
         if (v2.type() == nAttrs) {
             for (auto & i : *v2.attrs) {
-                completions->add(i.name);
+                std::string name = i.name;
+                if (name.find(searchWord) == 0) {
+                    completions->add(i.name);
+                }
             }
         }
     } else {

From 83615fcf8fe6095b60e33097e2d15fb1a3f56298 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 5 Jul 2021 13:09:46 +0200
Subject: [PATCH 76/90] Allow `fetchGit` to take a `name` argument

Fix #3388
---
 src/libfetchers/fetchers.cc | 7 ++++++-
 src/libfetchers/fetchers.hh | 2 ++
 src/libfetchers/git.cc      | 8 ++++----
 tests/fetchGit.sh           | 4 ++++
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 916e0a8e8..e158d914b 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -200,12 +200,17 @@ void Input::markChangedFile(
     return scheme->markChangedFile(*this, file, commitMsg);
 }
 
+std::string Input::getName() const
+{
+    return maybeGetStrAttr(attrs, "name").value_or("source");
+}
+
 StorePath Input::computeStorePath(Store & store) const
 {
     auto narHash = getNarHash();
     if (!narHash)
         throw Error("cannot compute store path for mutable input '%s'", to_string());
-    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
 }
 
 std::string Input::getType() const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index a72cfafa4..c839cf23b 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -81,6 +81,8 @@ public:
         std::string_view file,
         std::optional<std::string> commitMsg) const;
 
+    std::string getName() const;
+
     StorePath computeStorePath(Store & store) const;
 
     // Convenience functions for common attributes.
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index d8e0dbe0a..bc1930138 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -60,7 +60,7 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -167,10 +167,10 @@ struct GitInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        std::string name = input.getName();
+
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
         bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
@@ -270,7 +270,7 @@ struct GitInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 // FIXME: maybe we should use the timestamp of the last
                 // modified dirty file?
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 88744ee7f..89294d8d2 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -189,3 +189,7 @@ path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rep
 rev4=$(git -C $repo rev-parse HEAD)
 rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).rev")
 [[ $rev4 = $rev4_nix ]]
+
+# The name argument should be handled
+path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; name = \"foo\"; }).outPath")
+[[ $path9 =~ -foo$ ]]

From a487a652ed2f662782529d8f2bea4cf90a05e0c9 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 6 Jul 2021 08:42:47 +0200
Subject: [PATCH 77/90] allow fetchTarball to take a `name` argument

---
 src/libfetchers/tarball.cc | 4 ++--
 tests/tarball.sh           | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 257465bae..031ccc5f7 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -196,7 +196,7 @@ struct TarballInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
+            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
                 throw Error("unsupported tarball input attribute '%s'", name);
 
         Input input;
@@ -226,7 +226,7 @@ struct TarballInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
         return {std::move(tree), input};
     }
 };
diff --git a/tests/tarball.sh b/tests/tarball.sh
index d53ec8cd9..279ef5672 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -30,6 +30,7 @@ test_tarball() {
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; })"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
 
     nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2

From e4b082a52b4cd983b50037783a647cf39cd18bdf Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 6 Jul 2021 08:43:06 +0200
Subject: [PATCH 78/90] allow fetchMercurial to take a `name` argument

---
 src/libexpr/primops/fetchMercurial.cc | 1 +
 src/libfetchers/mercurial.cc          | 8 ++++----
 tests/fetchMercurial.sh               | 5 +++++
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 4830ebec3..3f88ccb91 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -62,6 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     fetchers::Attrs attrs;
     attrs.insert_or_assign("type", "hg");
     attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    attrs.insert_or_assign("name", name);
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
     auto input = fetchers::Input::fromAttrs(std::move(attrs));
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 0eb401e10..efb4ee8db 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -74,7 +74,7 @@ struct MercurialInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "hg") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
                 throw Error("unsupported Mercurial input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -147,10 +147,10 @@ struct MercurialInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        auto name = input.getName();
+
         auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
@@ -193,7 +193,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 return {
                     Tree(store->toRealPath(storePath), std::move(storePath)),
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index d8a4e09d2..726840664 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -94,3 +94,8 @@ hg commit --cwd $repo -m 'Bla3'
 
 path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
 [[ $path2 = $path4 ]]
+
+echo paris > $repo/hello
+# Passing a `name` argument should be reflected in the output path
+path5=$(nix eval -vvvvv --impure --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; name = \"foo\"; } ).outPath")
+[[ $path5 =~ -foo$ ]]

From 2c8240677edb94f17cc0bcd681e442c46b987013 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Tue, 6 Jul 2021 08:43:19 +0200
Subject: [PATCH 79/90] allow fetchFromGitHub to take a `name` argument

---
 src/libfetchers/github.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8352ef02d..298c05f9a 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+        auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
 
         input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
 

From 3b3e6bb1e50fae42ab509b0e3910c93b83dc84cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jul 2021 10:02:55 +0200
Subject: [PATCH 80/90] Style tweaks

---
 src/libfetchers/registry.cc |  2 +-
 src/libfetchers/registry.hh |  2 +-
 src/nix/registry-pin.md     |  2 +-
 src/nix/registry.cc         | 11 ++++++-----
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index c2dfa9dc6..f35359d4b 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -124,7 +124,7 @@ std::shared_ptr<Registry> getUserRegistry()
     return userRegistry;
 }
 
-std::shared_ptr<Registry> getCustomRegistry(Path p)
+std::shared_ptr<Registry> getCustomRegistry(const Path & p)
 {
     static auto customRegistry =
         Registry::read(p, Registry::Custom);
diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh
index cde22dff6..260a2c460 100644
--- a/src/libfetchers/registry.hh
+++ b/src/libfetchers/registry.hh
@@ -49,7 +49,7 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
 
 std::shared_ptr<Registry> getUserRegistry();
 
-std::shared_ptr<Registry> getCustomRegistry(Path p);
+std::shared_ptr<Registry> getCustomRegistry(const Path & p);
 
 Path getUserRegistryPath();
 
diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md
index 7b163c463..ebc0e3eff 100644
--- a/src/nix/registry-pin.md
+++ b/src/nix/registry-pin.md
@@ -24,7 +24,7 @@ R""(
   …
   ```
 
-* Pin `nixpkgs` in a custom registry to its most recent Git revision
+* Pin `nixpkgs` in a custom registry to its most recent Git revision:
 
   ```console
   # nix registry pin --registry ./custom-flake-registry.json nixpkgs
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index ec80dc0bc..6a92576c7 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -11,13 +11,12 @@ using namespace nix;
 using namespace nix::flake;
 
 
-class RegistryCommand: virtual Args
+class RegistryCommand : virtual Args
 {
     std::string registry_path;
 
     std::shared_ptr<fetchers::Registry> registry;
 
-
 public:
 
     RegistryCommand()
@@ -30,7 +29,8 @@ public:
         });
     }
 
-    std::shared_ptr<fetchers::Registry> getRegistry() {
+    std::shared_ptr<fetchers::Registry> getRegistry()
+    {
         if (registry) return registry;
         if (registry_path.empty()) {
             registry = fetchers::getUserRegistry();
@@ -40,7 +40,8 @@ public:
         return registry;
     }
 
-    Path getRegistryPath() {
+    Path getRegistryPath()
+    {
         if (registry_path.empty()) {
             return fetchers::getUserRegistryPath();
         } else {
@@ -156,7 +157,7 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand
 
     std::string description() override
     {
-        return "pin a flake to its current version in user flake registry or to the current version of a flake URI";
+        return "pin a flake to its current version or to the current version of a flake URL";
     }
 
     std::string doc() override

From 02dd6bb610e55a009cd7a4c83639698d3a7acaa2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jul 2021 10:48:47 +0200
Subject: [PATCH 81/90] tests/check.sh: Fix a race

Fixes this random failure:

    error: hash mismatch in fixed-output derivation '/tmp/nix-shell.EUgAVU/nix-test/tests/check/store/sfps3l3c5n7dabpx34kigxnfhmrwk2h6-dummy.drv':
             specified: sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=
                got:    sha256-0qhPS4tlCTfsj3PNi+LHSt1akRumTfJ0WO2CKdqASiY=

which happens because multiple tests were writing to ./dummy.
---
 tests/add.sh    | 2 +-
 tests/check.nix | 2 +-
 tests/check.sh  | 5 +++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/add.sh b/tests/add.sh
index e26e05843..5c3eed793 100644
--- a/tests/add.sh
+++ b/tests/add.sh
@@ -9,7 +9,7 @@ echo $path2
 if test "$path1" != "$path2"; then
     echo "nix-store --add and --add-fixed mismatch"
     exit 1
-fi    
+fi
 
 path3=$(nix-store --add-fixed sha256 ./dummy)
 echo $path3
diff --git a/tests/check.nix b/tests/check.nix
index bca04fdaf..ec455ae2d 100644
--- a/tests/check.nix
+++ b/tests/check.nix
@@ -44,7 +44,7 @@ with import ./config.nix;
   };
 
   hashmismatch = import <nix/fetchurl.nix> {
-    url = "file://" + toString ./dummy;
+    url = "file://" + builtins.getEnv "TMPDIR" + "/dummy";
     sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73";
   };
 
diff --git a/tests/check.sh b/tests/check.sh
index 5f4997e28..d26d4d8fc 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -74,12 +74,13 @@ nix-build check.nix -A fetchurl --no-out-link --check
 nix-build check.nix -A fetchurl --no-out-link --repair
 [[ $(cat $path) != foo ]]
 
+echo 'Hello World' > $TMPDIR/dummy
 nix-build check.nix -A hashmismatch --no-out-link || status=$?
 [ "$status" = "102" ]
 
-echo -n > ./dummy
+echo -n > $TMPDIR/dummy
 nix-build check.nix -A hashmismatch --no-out-link
-echo 'Hello World' > ./dummy
+echo 'Hello World' > $TMPDIR/dummy
 
 nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
 [ "$status" = "102" ]

From 6060ea1b0e4346a632b218d74b68ab18c20e100e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jul 2021 11:30:35 +0200
Subject: [PATCH 82/90] Add tests/dummy

---
 .gitignore    | 1 -
 tests/dummy   | 1 +
 tests/init.sh | 2 --
 3 files changed, 1 insertion(+), 3 deletions(-)
 create mode 100644 tests/dummy

diff --git a/.gitignore b/.gitignore
index 2e14561fe..5f402dbc3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -76,7 +76,6 @@ perl/Makefile.config
 # /tests/
 /tests/test-tmp
 /tests/common.sh
-/tests/dummy
 /tests/result*
 /tests/restricted-innocent
 /tests/shell
diff --git a/tests/dummy b/tests/dummy
new file mode 100644
index 000000000..557db03de
--- /dev/null
+++ b/tests/dummy
@@ -0,0 +1 @@
+Hello World
diff --git a/tests/init.sh b/tests/init.sh
index 1a6ccb6fe..6e45a939f 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -35,5 +35,3 @@ nix-store --init
 
 # Did anything happen?
 test -e "$NIX_STATE_DIR"/db/db.sqlite
-
-echo 'Hello World' > ./dummy

From 7e5c79a2d210e7b986803d5fe5c7ebb504b3e74e Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 8 Jul 2021 13:29:58 +0200
Subject: [PATCH 83/90] Forbid the `name` attribute for fetchTree
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We need to support it for the “old” fetch* functions for backwards
compatibility, but we don’t need it for fetchTree (as it’s a new
function).
Given that changing the `name` messes-up the content hashing, we can
just forbid passing a custom `name` argument to it
---
 src/libexpr/primops/fetchTree.cc | 7 +++++++
 tests/tarball.sh                 | 6 +++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index b8b99d4fa..472efe0ee 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -119,6 +119,13 @@ static void fetchTree(
                 .errPos = pos
             });
 
+        if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
+            throw Error({
+                .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+                .errPos = pos
+            });
+
+
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
         auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
diff --git a/tests/tarball.sh b/tests/tarball.sh
index 279ef5672..1301922a5 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -30,7 +30,6 @@ test_tarball() {
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
-    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; })"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
 
     nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
@@ -41,6 +40,11 @@ test_tarball() {
     (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)
 
     nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball$ext -I fnord=.
+
+    # Ensure that the `name` attribute isn’t accepted as that would mess
+    # with the content-addressing
+    (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; }")
+
 }
 
 test_tarball '' cat

From a654c1d81c63f13a49591436d6638d7c42222b99 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 8 Jul 2021 14:57:48 +0200
Subject: [PATCH 84/90] Restore the possibility to use a `name` parameter in
 fetchGit

Accidentally broken by 7e5c79a2d210e7b986803d5fe5c7ebb504b3e74e
---
 src/libexpr/primops/fetchTree.cc | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 472efe0ee..a57bda04e 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -72,13 +72,18 @@ void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v
     attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
 }
 
+struct FetchTreeParams {
+    bool emptyRevFallback = false;
+    bool allowNameArgument = false;
+};
+
 static void fetchTree(
     EvalState &state,
     const Pos &pos,
     Value **args,
     Value &v,
     const std::optional<std::string> type,
-    bool emptyRevFallback = false
+    const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
     PathSet context;
@@ -119,11 +124,12 @@ static void fetchTree(
                 .errPos = pos
             });
 
-        if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
-            throw Error({
-                .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
-                .errPos = pos
-            });
+        if (!params.allowNameArgument)
+            if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
+                throw Error({
+                    .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+                    .errPos = pos
+                });
 
 
         input = fetchers::Input::fromAttrs(std::move(attrs));
@@ -151,13 +157,13 @@ static void fetchTree(
     if (state.allowedPaths)
         state.allowedPaths->insert(tree.actualPath);
 
-    emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
+    emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback);
 }
 
 static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     settings.requireExperimentalFeature("flakes");
-    fetchTree(state, pos, args, v, std::nullopt);
+    fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
 }
 
 // FIXME: document
@@ -299,7 +305,7 @@ static RegisterPrimOp primop_fetchTarball({
 
 static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
 {
-    fetchTree(state, pos, args, v, "git", true);
+    fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
 }
 
 static RegisterPrimOp primop_fetchGit({

From b1cfe8f984145c836c1dab229c1117d6396a4305 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Jul 2021 17:12:01 +0200
Subject: [PATCH 85/90] nix develop: Add a test for arrays

---
 tests/nix-shell.sh | 1 +
 tests/shell.nix    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 3481c2c69..aae7cbcde 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -98,3 +98,4 @@ nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
 # Test 'nix print-dev-env'.
 source <(nix print-dev-env -f shell.nix shellDrv)
 [[ -n $stdenv ]]
+[[ ${arr1[2]} = "3 4" ]]
diff --git a/tests/shell.nix b/tests/shell.nix
index f174db583..42d2d2033 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -20,6 +20,7 @@ let pkgs = rec {
     for pkg in $buildInputs; do
       export PATH=$PATH:$pkg/bin
     done
+    declare -a arr1=(1 2 "3 4" 5)
   '';
 
   stdenv = mkDerivation {

From b1f1347ade81d1f04f2d490baceefb3c4de0b4e3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Jul 2021 00:47:57 +0200
Subject: [PATCH 86/90] nix develop: Don't parse bash environment with regexes

Instead have get-env.sh dump the bash environment as JSON. This should
be a lot less error-prone.

Fixes #4992.
---
 src/nix/develop.cc | 176 ++++++++++++++++++++++-----------------------
 src/nix/get-env.sh |  92 +++++++++++++++++++++++-
 2 files changed, 174 insertions(+), 94 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 699ec0b99..e00f0d575 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -8,7 +8,7 @@
 #include "affinity.hh"
 #include "progress-bar.hh"
 
-#include <regex>
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -25,94 +25,98 @@ static DevelopSettings developSettings;
 
 static GlobalConfig::Register rDevelopSettings(&developSettings);
 
-struct Var
-{
-    bool exported = true;
-    bool associative = false;
-    std::string quoted; // quoted string or array
-};
-
 struct BuildEnvironment
 {
-    std::map<std::string, Var> env;
-    std::string bashFunctions;
-};
+    struct String
+    {
+        bool exported;
+        std::string value;
+    };
 
-BuildEnvironment readEnvironment(const Path & path)
-{
-    BuildEnvironment res;
+    using Array = std::vector<std::string>;
 
-    std::set<std::string> exported;
+    using Associative = std::map<std::string, std::string>;
 
-    debug("reading environment file '%s'", path);
+    using Value = std::variant<String, Array, Associative>;
 
-    auto file = readFile(path);
+    std::map<std::string, Value> vars;
+    std::map<std::string, std::string> bashFunctions;
 
-    auto pos = file.cbegin();
+    static BuildEnvironment fromJSON(const Path & path)
+    {
+        BuildEnvironment res;
 
-    static std::string varNameRegex =
-        R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
+        std::set<std::string> exported;
 
-    static std::string simpleStringRegex =
-        R"re((?:[a-zA-Z0-9_/:\.\-\+=@%]*))re";
+        debug("reading environment file '%s'", path);
 
-    static std::string dquotedStringRegex =
-        R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
+        auto json = nlohmann::json::parse(readFile(path));
 
-    static std::string squotedStringRegex =
-        R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re";
-
-    static std::string indexedArrayRegex =
-        R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
-
-    static std::regex declareRegex(
-        "^declare -a?x (" + varNameRegex + ")(=(" +
-        dquotedStringRegex + "|" + indexedArrayRegex + "))?\n");
-
-    static std::regex varRegex(
-        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n");
-
-    /* Note: we distinguish between an indexed and associative array
-       using the space before the closing parenthesis. Will
-       undoubtedly regret this some day. */
-    static std::regex assocArrayRegex(
-        "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n");
-
-    static std::regex functionRegex(
-        "^" + varNameRegex + " \\(\\) *\n");
-
-    while (pos != file.end()) {
-
-        std::smatch match;
-
-        if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            exported.insert(match[1]);
+        for (auto & [name, info] : json["variables"].items()) {
+            std::string type = info["type"];
+            if (type == "var" || type == "exported")
+                res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }});
+            else if (type == "array")
+                res.vars.insert({name, (Array) info["value"]});
+            else if (type == "associative")
+                res.vars.insert({name, (Associative) info["value"]});
         }
 
-        else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
+        for (auto & [name, def] : json["bashFunctions"].items()) {
+            res.bashFunctions.insert({name, def});
         }
 
-        else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
-        }
-
-        else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
-            res.bashFunctions = std::string(pos, file.cend());
-            break;
-        }
-
-        else throw Error("shell environment '%s' has unexpected line '%s'",
-            path, file.substr(pos - file.cbegin(), 60));
+        return res;
     }
 
-    res.env.erase("__output");
+    void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
+    {
+        for (auto & [name, value] : vars) {
+            if (!ignoreVars.count(name) && !hasPrefix(name, "BASH_")) {
+                if (auto str = std::get_if<String>(&value)) {
+                    out << fmt("%s=%s\n", name, shellEscape(str->value));
+                    if (str->exported)
+                        out << fmt("export %s\n", name);
+                }
+                else if (auto arr = std::get_if<Array>(&value)) {
+                    out << "declare -a " << name << "=(";
+                    for (auto & s : *arr)
+                        out << shellEscape(s) << " ";
+                    out << ")\n";
+                }
+                else if (auto arr = std::get_if<Associative>(&value)) {
+                    out << "declare -A " << name << "=(";
+                    for (auto & [n, v] : *arr)
+                        out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " ";
+                    out << ")\n";
+                }
+            }
+        }
 
-    return res;
-}
+        for (auto & [name, def] : bashFunctions) {
+            out << name << " ()\n{\n" << def << "}\n";
+        }
+    }
+
+    static std::string getString(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return str->value;
+        else
+            throw Error("bash variable is not a string");
+    }
+
+    static Array getStrings(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return tokenizeString<Array>(str->value);
+        else if (auto arr = std::get_if<Array>(&value)) {
+            return *arr;
+        }
+        else
+            throw Error("bash variable is not a string or array");
+    }
+};
 
 const static std::string getEnvSh =
     #include "get-env.sh.gen.hh"
@@ -185,7 +189,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 
 struct Common : InstallableCommand, MixProfile
 {
-    std::set<string> ignoreVars{
+    std::set<std::string> ignoreVars{
         "BASHOPTS",
         "EUID",
         "HOME", // FIXME: don't ignore in pure mode?
@@ -233,22 +237,10 @@ struct Common : InstallableCommand, MixProfile
 
         out << "nix_saved_PATH=\"$PATH\"\n";
 
-        for (auto & i : buildEnvironment.env) {
-            if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
-                if (i.second.associative)
-                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
-                else {
-                    out << fmt("%s=%s\n", i.first, i.second.quoted);
-                    if (i.second.exported)
-                        out << fmt("export %s\n", i.first);
-                }
-            }
-        }
+        buildEnvironment.toBash(out, ignoreVars);
 
         out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
 
-        out << buildEnvironment.bashFunctions << "\n";
-
         out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
@@ -258,16 +250,16 @@ struct Common : InstallableCommand, MixProfile
         auto script = out.str();
 
         /* Substitute occurrences of output paths. */
-        auto outputs = buildEnvironment.env.find("outputs");
-        assert(outputs != buildEnvironment.env.end());
+        auto outputs = buildEnvironment.vars.find("outputs");
+        assert(outputs != buildEnvironment.vars.end());
 
         // FIXME: properly unquote 'outputs'.
         StringMap rewrites;
-        for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
-            auto from = buildEnvironment.env.find(outputName);
-            assert(from != buildEnvironment.env.end());
+        for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) {
+            auto from = buildEnvironment.vars.find(outputName);
+            assert(from != buildEnvironment.vars.end());
             // FIXME: unquote
-            rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
+            rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName});
         }
 
         /* Substitute redirects. */
@@ -321,7 +313,7 @@ struct Common : InstallableCommand, MixProfile
 
         updateProfile(shellOutPath);
 
-        return {readEnvironment(strPath), strPath};
+        return {BuildEnvironment::fromJSON(strPath), strPath};
     }
 };
 
diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
index 091c0f573..834b84e35 100644
--- a/src/nix/get-env.sh
+++ b/src/nix/get-env.sh
@@ -8,10 +8,98 @@ if [[ -n $stdenv ]]; then
     source $stdenv/setup
 fi
 
+# Better to use compgen, but stdenv bash doesn't have it.
+__vars="$(declare -p)"
+__functions="$(declare -F)"
+
+__dumpEnv() {
+    printf '{\n'
+
+    printf '  "bashFunctions": {\n'
+    local __first=1
+    while read __line; do
+        if ! [[ $__line =~ ^declare\ -f\ (.*) ]]; then continue; fi
+        __fun_name="${BASH_REMATCH[1]}"
+        __fun_body="$(type $__fun_name)"
+        if [[ $__fun_body =~ \{(.*)\} ]]; then
+            if [[ -z $__first ]]; then printf ',\n'; else __first=; fi
+            __fun_body="${BASH_REMATCH[1]}"
+            printf "    "
+            __escapeString "$__fun_name"
+            printf ':'
+            __escapeString "$__fun_body"
+        else
+            printf "Cannot parse definition of function '%s'.\n" "$__fun_name" >&2
+            return 1
+        fi
+    done < <(printf "%s\n" "$__functions")
+    printf '\n  },\n'
+
+    printf '  "variables": {\n'
+    local __first=1
+    while read __line; do
+        if ! [[ $__line =~ ^declare\ (-[^ ])\ ([^=]*) ]]; then continue; fi
+        local type="${BASH_REMATCH[1]}"
+        local __var_name="${BASH_REMATCH[2]}"
+
+        if [[ -z $__first ]]; then printf ',\n'; else __first=; fi
+
+        printf "    "
+        __escapeString "$__var_name"
+        printf ': {'
+
+        # FIXME: handle -i, -r, -n.
+        if [[ $type == -x ]]; then
+            printf '"type": "exported", "value": '
+            __escapeString "${!__var_name}"
+        elif [[ $type == -- ]]; then
+            printf '"type": "var", "value": '
+            __escapeString "${!__var_name}"
+        elif [[ $type == -a ]]; then
+            printf '"type": "array", "value": ['
+            local __first2=1
+            __var_name="$__var_name[@]"
+            for __i in "${!__var_name}"; do
+                if [[ -z $__first2 ]]; then printf ', '; else __first2=; fi
+                __escapeString "$__i"
+                printf ' '
+            done
+            printf ']'
+        elif [[ $type == -A ]]; then
+            printf '"type": "associative", "value": {\n'
+            local __first2=1
+            declare -n __var_name2="$__var_name"
+            for __i in "${!__var_name2[@]}"; do
+                if [[ -z $__first2 ]]; then printf ',\n'; else __first2=; fi
+                printf "      "
+                __escapeString "$__i"
+                printf ": "
+                __escapeString "${__var_name2[$__i]}"
+            done
+            printf '\n    }'
+        else
+            printf '"type": "unknown"'
+        fi
+
+        printf "}"
+    done < <(printf "%s\n" "$__vars")
+    printf '\n  }\n}'
+}
+
+__escapeString() {
+    local __s="$1"
+    __s="${__s//\\/\\\\}"
+    __s="${__s//\"/\\\"}"
+    __s="${__s//$'\n'/\\n}"
+    __s="${__s//$'\r'/\\r}"
+    __s="${__s//$'\t'/\\t}"
+    printf '"%s"' "$__s"
+}
+
+# Dump the bash environment as JSON.
 for __output in $outputs; do
     if [[ -z $__done ]]; then
-        export > ${!__output}
-        set >> ${!__output}
+        __dumpEnv > ${!__output}
         __done=1
     else
         echo -n >> ${!__output}

From 5f6375a816803d17c8ec0f7c2ef960c848be6166 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Jul 2021 01:02:47 +0200
Subject: [PATCH 87/90] nix develop: Filter some bash magic variables

---
 src/nix/develop.cc |  6 +-----
 src/nix/get-env.sh | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index e00f0d575..9b2945ba3 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -72,7 +72,7 @@ struct BuildEnvironment
     void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
     {
         for (auto & [name, value] : vars) {
-            if (!ignoreVars.count(name) && !hasPrefix(name, "BASH_")) {
+            if (!ignoreVars.count(name)) {
                 if (auto str = std::get_if<String>(&value)) {
                     out << fmt("%s=%s\n", name, shellEscape(str->value));
                     if (str->exported)
@@ -191,17 +191,13 @@ struct Common : InstallableCommand, MixProfile
 {
     std::set<std::string> ignoreVars{
         "BASHOPTS",
-        "EUID",
         "HOME", // FIXME: don't ignore in pure mode?
-        "HOSTNAME",
         "NIX_BUILD_TOP",
         "NIX_ENFORCE_PURITY",
         "NIX_LOG_FD",
         "NIX_REMOTE",
         "PPID",
-        "PWD",
         "SHELLOPTS",
-        "SHLVL",
         "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
         "TEMP",
         "TEMPDIR",
diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
index 834b84e35..20937b956 100644
--- a/src/nix/get-env.sh
+++ b/src/nix/get-env.sh
@@ -42,6 +42,20 @@ __dumpEnv() {
         local type="${BASH_REMATCH[1]}"
         local __var_name="${BASH_REMATCH[2]}"
 
+        if [[ $__var_name =~ ^BASH_ || \
+              $__var_name = _ || \
+              $__var_name = DIRSTACK || \
+              $__var_name = EUID || \
+              $__var_name = FUNCNAME || \
+              $__var_name = HISTCMD || \
+              $__var_name = HOSTNAME || \
+              $__var_name = PIPESTATUS || \
+              $__var_name = PWD || \
+              $__var_name = RANDOM || \
+              $__var_name = SHLVL || \
+              $__var_name = SECONDS \
+            ]]; then continue; fi
+
         if [[ -z $__first ]]; then printf ',\n'; else __first=; fi
 
         printf "    "

From 9fc7da1e08d83378f8b0d66a060d92929c1468cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Jul 2021 01:16:55 +0200
Subject: [PATCH 88/90] Add test for #4992

---
 tests/nix-shell.sh | 2 ++
 tests/shell.nix    | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index aae7cbcde..728313a8a 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -99,3 +99,5 @@ nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
 source <(nix print-dev-env -f shell.nix shellDrv)
 [[ -n $stdenv ]]
 [[ ${arr1[2]} = "3 4" ]]
+[[ ${arr2[1]} = $'\n' ]]
+[[ ${arr2[2]} = $'x\ny' ]]
diff --git a/tests/shell.nix b/tests/shell.nix
index 42d2d2033..1f5526253 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -21,6 +21,7 @@ let pkgs = rec {
       export PATH=$PATH:$pkg/bin
     done
     declare -a arr1=(1 2 "3 4" 5)
+    declare -a arr2=(x $'\n' $'x\ny')
   '';
 
   stdenv = mkDerivation {

From e50408bd3192156e04571a1546d053f998ab4e2c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Jul 2021 01:17:32 +0200
Subject: [PATCH 89/90] nix develop: Add a test for bash functions

---
 tests/nix-shell.sh | 1 +
 tests/shell.nix    | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 728313a8a..ec43db349 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -101,3 +101,4 @@ source <(nix print-dev-env -f shell.nix shellDrv)
 [[ ${arr1[2]} = "3 4" ]]
 [[ ${arr2[1]} = $'\n' ]]
 [[ ${arr2[2]} = $'x\ny' ]]
+[[ $(fun) = blabla ]]
diff --git a/tests/shell.nix b/tests/shell.nix
index 1f5526253..70c60276c 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -22,6 +22,9 @@ let pkgs = rec {
     done
     declare -a arr1=(1 2 "3 4" 5)
     declare -a arr2=(x $'\n' $'x\ny')
+    fun() {
+      echo blabla
+    }
   '';
 
   stdenv = mkDerivation {

From 86fb01c4bee0c4f451504c0f6fd2066d2adc8fc2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Jul 2021 12:10:48 +0200
Subject: [PATCH 90/90] nix print-dev-env: Add --json flag

---
 src/nix/develop.cc       | 59 +++++++++++++++++++++++++++++++++++-----
 src/nix/get-env.sh       |  1 +
 src/nix/print-dev-env.md | 37 +++++++++++++++++++++++--
 tests/nix-shell.sh       |  2 ++
 4 files changed, 89 insertions(+), 10 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 9b2945ba3..ca04f297d 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -31,6 +31,11 @@ struct BuildEnvironment
     {
         bool exported;
         std::string value;
+
+        bool operator == (const String & other) const
+        {
+            return exported == other.exported && value == other.value;
+        }
     };
 
     using Array = std::vector<std::string>;
@@ -42,15 +47,13 @@ struct BuildEnvironment
     std::map<std::string, Value> vars;
     std::map<std::string, std::string> bashFunctions;
 
-    static BuildEnvironment fromJSON(const Path & path)
+    static BuildEnvironment fromJSON(std::string_view in)
     {
         BuildEnvironment res;
 
         std::set<std::string> exported;
 
-        debug("reading environment file '%s'", path);
-
-        auto json = nlohmann::json::parse(readFile(path));
+        auto json = nlohmann::json::parse(in);
 
         for (auto & [name, info] : json["variables"].items()) {
             std::string type = info["type"];
@@ -69,6 +72,38 @@ struct BuildEnvironment
         return res;
     }
 
+    std::string toJSON() const
+    {
+        auto res = nlohmann::json::object();
+
+        auto vars2 = nlohmann::json::object();
+        for (auto & [name, value] : vars) {
+            auto info = nlohmann::json::object();
+            if (auto str = std::get_if<String>(&value)) {
+                info["type"] = str->exported ? "exported" : "var";
+                info["value"] = str->value;
+            }
+            else if (auto arr = std::get_if<Array>(&value)) {
+                info["type"] = "array";
+                info["value"] = *arr;
+            }
+            else if (auto arr = std::get_if<Associative>(&value)) {
+                info["type"] = "associative";
+                info["value"] = *arr;
+            }
+            vars2[name] = std::move(info);
+        }
+        res["variables"] = std::move(vars2);
+
+        res["bashFunctions"] = bashFunctions;
+
+        auto json = res.dump();
+
+        assert(BuildEnvironment::fromJSON(json) == *this);
+
+        return json;
+    }
+
     void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
     {
         for (auto & [name, value] : vars) {
@@ -116,6 +151,11 @@ struct BuildEnvironment
         else
             throw Error("bash variable is not a string or array");
     }
+
+    bool operator == (const BuildEnvironment & other) const
+    {
+        return vars == other.vars && bashFunctions == other.bashFunctions;
+    }
 };
 
 const static std::string getEnvSh =
@@ -309,7 +349,9 @@ struct Common : InstallableCommand, MixProfile
 
         updateProfile(shellOutPath);
 
-        return {BuildEnvironment::fromJSON(strPath), strPath};
+        debug("reading environment file '%s'", strPath);
+
+        return {BuildEnvironment::fromJSON(readFile(strPath)), strPath};
     }
 };
 
@@ -462,7 +504,7 @@ struct CmdDevelop : Common, MixEnvironment
     }
 };
 
-struct CmdPrintDevEnv : Common
+struct CmdPrintDevEnv : Common, MixJSON
 {
     std::string description() override
     {
@@ -484,7 +526,10 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        std::cout << makeRcScript(store, buildEnvironment);
+        logger->writeToStdout(
+            json
+            ? buildEnvironment.toJSON()
+            : makeRcScript(store, buildEnvironment));
     }
 };
 
diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
index 20937b956..2df3e543f 100644
--- a/src/nix/get-env.sh
+++ b/src/nix/get-env.sh
@@ -49,6 +49,7 @@ __dumpEnv() {
               $__var_name = FUNCNAME || \
               $__var_name = HISTCMD || \
               $__var_name = HOSTNAME || \
+              $__var_name = GROUPS || \
               $__var_name = PIPESTATUS || \
               $__var_name = PWD || \
               $__var_name = RANDOM || \
diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md
index b80252acf..2aad491de 100644
--- a/src/nix/print-dev-env.md
+++ b/src/nix/print-dev-env.md
@@ -8,12 +8,43 @@ R""(
   # . <(nix print-dev-env nixpkgs#hello)
   ```
 
+* Get the build environment in JSON format:
+
+  ```console
+  # nix print-dev-env nixpkgs#hello --json
+  ```
+
+  The output will look like this:
+
+  ```json
+  {
+    "bashFunctions": {
+      "buildPhase": " \n    runHook preBuild;\n...",
+      ...
+    },
+    "variables": {
+      "src": {
+        "type": "exported",
+        "value": "/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz"
+      },
+      "postUnpackHooks": {
+        "type": "array",
+        "value": ["_updateSourceDateEpochFromSourceRoot"]
+      },
+      ...
+    }
+  }
+  ```
+
 # Description
 
-This command prints a shell script that can be sourced by `b`ash and
-that sets the environment variables and shell functions defined by the
-build process of *installable*. This allows you to get a similar build
+This command prints a shell script that can be sourced by `bash` and
+that sets the variables and shell functions defined by the build
+process of *installable*. This allows you to get a similar build
 environment in your current shell rather than in a subshell (as with
 `nix develop`).
 
+With `--json`, the output is a JSON serialisation of the variables and
+functions defined by the build process.
+
 )""
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index ec43db349..f60102f9c 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -96,6 +96,8 @@ echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
 nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
 
 # Test 'nix print-dev-env'.
+[[ $(nix print-dev-env -f shell.nix shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]]
+
 source <(nix print-dev-env -f shell.nix shellDrv)
 [[ -n $stdenv ]]
 [[ ${arr1[2]} = "3 4" ]]