From 836573a9a2d38935e254702826d250ea39824a1b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Oct 2017 12:22:29 +0100
Subject: [PATCH 001/112] Dynamically allocate UIDs

Rather than rely on a nixbld group, we now allocate UIDs/GIDs
dynamically starting at a configurable ID (872415232 by default).

Also, we allocate 2^18 UIDs and GIDs per build, and run the build as
root in its UID namespace. (This should not be the default since it
breaks some builds. We probably should enable this conditional on a
requiredSystemFeature.) The goal is to be able to run (NixOS)
containers in a build. However, this will also require some cgroup
initialisation.

The 2^18 UIDs/GIDs is intended to provide enough ID space to run
multiple containers per build, e.g. for distributed NixOS tests.
---
 src/libstore/build.cc   | 59 +++++++++++++++++++++++++++++++++++------
 src/libstore/globals.hh | 10 +++++++
 2 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 347fe1b99..c853f609d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -513,7 +513,6 @@ private:
     AutoCloseFD fdUserLock;
 
     bool isEnabled = false;
-    string user;
     uid_t uid = 0;
     gid_t gid = 0;
     std::vector<gid_t> supplementaryGIDs;
@@ -523,9 +522,9 @@ public:
 
     void kill();
 
-    string getUser() { return user; }
     uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
+    gid_t getGID() { assert(gid); return gid; }
+    uint32_t getIDCount() { return 1; }
     std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
 
     bool findFreeUser();
@@ -537,13 +536,16 @@ public:
 
 UserLock::UserLock()
 {
+#if 0
     assert(settings.buildUsersGroup != "");
     createDirs(settings.nixStateDir + "/userpool");
+#endif
 }
 
 bool UserLock::findFreeUser() {
     if (enabled()) return true;
 
+#if 0
     /* Get the members of the build-users-group. */
     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
     if (!gr)
@@ -607,12 +609,46 @@ bool UserLock::findFreeUser() {
         }
     }
 
+    return false;
+#endif
+
+    assert(settings.startId > 0);
+    assert(settings.startId % settings.idsPerBuild == 0);
+    assert(settings.uidCount % settings.idsPerBuild == 0);
+    assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
+
+    // FIXME: check whether the id range overlaps any known users
+
+    size_t nrSlots = settings.uidCount / settings.idsPerBuild;
+
+    for (size_t i = 0; i < nrSlots; i++) {
+        debug("trying user slot '%d'", i);
+
+        createDirs(settings.nixStateDir + "/userpool");
+
+        fnUserLock = fmt("%s/userpool/slot-%d", settings.nixStateDir, i);
+
+        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+        if (!fd)
+            throw SysError("opening user lock '%1%'", fnUserLock);
+
+        if (lockFile(fd.get(), ltWrite, false)) {
+            fdUserLock = std::move(fd);
+            uid = settings.startId + i * settings.idsPerBuild;
+            gid = settings.startId + i * settings.idsPerBuild;
+            return true;
+        }
+    }
+
     return false;
 }
 
 void UserLock::kill()
 {
+    // FIXME: use a cgroup to kill all processes in the build?
+#if 0
     killUser(uid);
+#endif
 }
 
 
@@ -1523,7 +1559,7 @@ void DerivationGoal::tryLocalBuild() {
 
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
+    if ((settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0) {
 #if defined(__linux__) || defined(__APPLE__)
         if (!buildUser) buildUser = std::make_unique<UserLock>();
 
@@ -2129,7 +2165,7 @@ void DerivationGoal::startBuilder()
 
         printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
 
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
+        if (mkdir(chrootRootDir.c_str(), 0755) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
         if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
@@ -2444,14 +2480,15 @@ void DerivationGoal::startBuilder()
            the calling user (if build users are disabled). */
         uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
         uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
+        uint32_t nrIds = settings.idsPerBuild; // FIXME
 
         writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-            (format("%d %d 1") % sandboxUid % hostUid).str());
+            fmt("%d %d %d", /* sandboxUid */ 0, hostUid, nrIds));
 
-        writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+        //writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
 
         writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-            (format("%d %d 1") % sandboxGid % hostGid).str());
+            fmt("%d %d %d", /* sandboxGid */ 0, hostGid, nrIds));
 
         /* Save the mount namespace of the child. We have to do this
            *before* the child does a chroot. */
@@ -3306,10 +3343,16 @@ void DerivationGoal::runChild()
             /* Switch to the sandbox uid/gid in the user namespace,
                which corresponds to the build user or calling user in
                the parent namespace. */
+#if 0
             if (setgid(sandboxGid) == -1)
                 throw SysError("setgid failed");
             if (setuid(sandboxUid) == -1)
                 throw SysError("setuid failed");
+#endif
+            if (setgid(0) == -1)
+                throw SysError("setgid failed");
+            if (setuid(0) == -1)
+                throw SysError("setuid failed");
 
             setUser = false;
         }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 58cf08763..7dc842bca 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -148,6 +148,16 @@ public:
     Setting<std::string> buildUsersGroup{this, "", "build-users-group",
         "The Unix group that contains the build users."};
 
+    #if __linux__
+    const uint32_t idsPerBuild = 1 << 18;
+
+    Setting<uint32_t> startId{this, 872415232, "start-id",
+        "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"};
+
+    Setting<uint32_t> uidCount{this, idsPerBuild * 128, "id-count",
+        "The number of UIDs/GIDs to use for dynamic ID allocation."};
+    #endif
+
     Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
         "Whether to impersonate a Linux 2.6 machine on newer kernels.",
         {"build-impersonate-linux-26"}};

From c3e0a68c7eeeb4f491c0464392b2146ddec4305a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 May 2020 13:52:41 +0200
Subject: [PATCH 002/112] canonicalisePathMetaData(): Support a UID range

---
 src/libstore/build.cc       | 12 +++++++++---
 src/libstore/local-store.cc | 27 +++++++++++++++++----------
 src/libstore/local-store.hh | 15 ++++++++++++---
 src/nix-store/nix-store.cc  |  2 +-
 4 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index c853f609d..4e654e8ad 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -524,7 +524,7 @@ public:
 
     uid_t getUID() { assert(uid); return uid; }
     gid_t getGID() { assert(gid); return gid; }
-    uint32_t getIDCount() { return 1; }
+    uint32_t getIDCount() { return settings.idsPerBuild; }
     std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
 
     bool findFreeUser();
@@ -3744,7 +3744,10 @@ void DerivationGoal::registerOutputs()
             /* Canonicalise first.  This ensures that the path we're
                rewriting doesn't contain a hard link to /etc/shadow or
                something like that. */
-            canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
+            canonicalisePathMetaData(
+                actualPath,
+                buildUser ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) : std::nullopt,
+                inodesSeen);
 
             /* FIXME: this is in-memory. */
             StringSink sink;
@@ -3819,7 +3822,10 @@ void DerivationGoal::registerOutputs()
         /* Get rid of all weird permissions.  This also checks that
            all files are owned by the build user, if applicable. */
         canonicalisePathMetaData(actualPath,
-            buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen);
+            buildUser && !rewritten
+            ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1))
+            : std::nullopt,
+            inodesSeen);
 
         /* For this output path, find the references to other paths
            contained in it.  Compute the SHA-256 NAR hash at the same
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index eed225349..80ebe903f 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -424,7 +424,10 @@ void canonicaliseTimestampAndPermissions(const Path & path)
 }
 
 
-static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
+static void canonicalisePathMetaData_(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen)
 {
     checkInterrupt();
 
@@ -475,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
        However, ignore files that we chown'ed ourselves previously to
        ensure that we don't fail on hard links within the same build
        (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
-    if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
+    if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
         assert(!S_ISDIR(st.st_mode));
         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
             throw BuildError("invalid ownership on file '%1%'", path);
@@ -509,14 +512,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     if (S_ISDIR(st.st_mode)) {
         DirEntries entries = readDirectory(path);
         for (auto & i : entries)
-            canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen);
+            canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
     }
 }
 
 
-void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen)
 {
-    canonicalisePathMetaData_(path, fromUid, inodesSeen);
+    canonicalisePathMetaData_(path, uidRange, inodesSeen);
 
     /* On platforms that don't have lchown(), the top-level path can't
        be a symlink, since we can't change its ownership. */
@@ -531,10 +537,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
 }
 
 
-void canonicalisePathMetaData(const Path & path, uid_t fromUid)
+void canonicalisePathMetaData(const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange)
 {
     InodesSeen inodesSeen;
-    canonicalisePathMetaData(path, fromUid, inodesSeen);
+    canonicalisePathMetaData(path, uidRange, inodesSeen);
 }
 
 
@@ -1021,7 +1028,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             autoGC();
 
-            canonicalisePathMetaData(realPath, -1);
+            canonicalisePathMetaData(realPath, {});
 
             optimisePath(realPath); // FIXME: combine with hashPath()
 
@@ -1064,7 +1071,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
             } else
                 writeFile(realPath, dump);
 
-            canonicalisePathMetaData(realPath, -1);
+            canonicalisePathMetaData(realPath, {});
 
             /* Register the SHA-256 hash of the NAR serialisation of
                the path in the database.  We may just have computed it
@@ -1134,7 +1141,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
             writeFile(realPath, s);
 
-            canonicalisePathMetaData(realPath, -1);
+            canonicalisePathMetaData(realPath, {});
 
             StringSink sink;
             dumpString(s, sink);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index ff36cb00e..79b415875 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -311,9 +311,18 @@ typedef set<Inode> InodesSeen;
    - the permissions are set of 444 or 555 (i.e., read-only with or
      without execute permission; setuid bits etc. are cleared)
    - the owner and group are set to the Nix user and group, if we're
-     running as root. */
-void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen);
-void canonicalisePathMetaData(const Path & path, uid_t fromUid);
+     running as root.
+   If uidRange is not empty, this function will throw an error if it
+   encounters files owned by a user outside of the closed interval
+   [uidRange->first, uidRange->second].
+*/
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen);
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange);
 
 void canonicaliseTimestampAndPermissions(const Path & path);
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 7d81bf54f..b948380bb 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -500,7 +500,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
         if (!store->isValidPath(info->path) || reregister) {
             /* !!! races */
             if (canonicalise)
-                canonicalisePathMetaData(store->printStorePath(info->path), -1);
+                canonicalisePathMetaData(store->printStorePath(info->path), {});
             if (!hashGiven) {
                 HashResult hash = hashPath(htSHA256, store->printStorePath(info->path));
                 info->narHash = hash.first;

From f5fa3de759a2b4c1d0107a4304a0b3f9571c87b6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 15 May 2020 00:11:59 +0200
Subject: [PATCH 003/112] Run builds in their own cgroup

Also, run builds in a cgroup namespace (ensuring /proc/self/cgroup
doesn't leak information about the outside world) and mount /sys. This
enables running systemd-nspawn and thus NixOS containers in a Nix
build.
---
 src/libstore/build.cc | 66 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4e654e8ad..816d695a5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2168,7 +2168,8 @@ void DerivationGoal::startBuilder()
         if (mkdir(chrootRootDir.c_str(), 0755) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
+        // FIXME: only make root writable for user namespace builds.
+        if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
             throw SysError("cannot change ownership of '%1%'", chrootRootDir);
 
         /* Create a writable /tmp in the chroot.  Many builders need
@@ -2182,6 +2183,7 @@ void DerivationGoal::startBuilder()
            nobody account.  The latter is kind of a hack to support
            Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
+        chownToBuilder(chrootRootDir + "/etc");
 
         writeFile(chrootRootDir + "/etc/passwd", fmt(
                 "root:x:0:0:Nix build user:%3%:/noshell\n"
@@ -2372,6 +2374,52 @@ void DerivationGoal::startBuilder()
 
 #if __linux__
     if (useChroot) {
+        /* Create a cgroup. */
+        // FIXME: do we want to use the parent cgroup? We should
+        // always use the same cgroup regardless of whether we're the
+        // daemon or run from a user session via sudo.
+        std::string msg;
+        std::vector<Path> cgroups;
+        for (auto & line : tokenizeString<std::vector<std::string>>(readFile("/proc/self/cgroup"), "\n")) {
+            static std::regex regex("([0-9]+):([^:]*):(.*)");
+            std::smatch match;
+            if (!std::regex_match(line, match, regex))
+                throw Error("invalid line '%s' in '/proc/self/cgroup'", line);
+
+            /* We only create a systemd cgroup, since that's enough
+               for running systemd-nspawn. */
+            std::string name;
+            if (match[2] == "name=systemd")
+                name = "systemd";
+            //else if (match[2] == "")
+            //    name = "unified";
+            else continue;
+
+            std::string cgroup = match[3];
+
+            auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup);
+
+            if (!pathExists(hostCgroup))
+                throw Error("expected unified cgroup directory '%s'", hostCgroup);
+
+            auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID());
+
+            // FIXME: if the cgroup already exists, kill all processes
+            // in it and destroy it.
+
+            if (mkdir(childCgroup.c_str(), 0755) == -1 && errno != EEXIST)
+                throw SysError("creating cgroup '%s'", childCgroup);
+
+            chownToBuilder(childCgroup);
+            chownToBuilder(childCgroup + "/cgroup.procs");
+            if (name == "unified") {
+                chownToBuilder(childCgroup + "/cgroup.threads");
+                chownToBuilder(childCgroup + "/cgroup.subtree_control");
+            }
+
+            cgroups.push_back(childCgroup);
+        }
+
         /* Set up private namespaces for the build:
 
            - The PID namespace causes the build to start as PID 1.
@@ -2496,6 +2544,10 @@ void DerivationGoal::startBuilder()
         if (sandboxMountNamespace.get() == -1)
             throw SysError("getting sandbox mount namespace");
 
+        /* Move the child into its own cgroup. */
+        for (auto & childCgroup : cgroups)
+            writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
+
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
         userNamespaceSync.writeSide = -1;
@@ -3279,6 +3331,12 @@ void DerivationGoal::runChild()
             if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
                 throw SysError("mounting /proc");
 
+            /* Mount sysfs on /sys. FIXME: only in user namespace
+               builds. */
+            createDirs(chrootRootDir + "/sys");
+            if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
+                throw SysError("mounting /sys");
+
             /* Mount a new tmpfs on /dev/shm to ensure that whatever
                the builder puts in /dev/shm is cleaned up automatically. */
             if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
@@ -3321,6 +3379,12 @@ void DerivationGoal::runChild()
             if (unshare(CLONE_NEWNS) == -1)
                 throw SysError("unsharing mount namespace");
 
+            /* Unshare the cgroup namespace. This means
+               /proc/self/cgroup will show the child's cgroup as '/'
+               rather than whatever it is in the parent. */
+            if (unshare(CLONE_NEWCGROUP) == -1)
+                throw SysError("unsharing cgroup namespace");
+
             /* Do the chroot(). */
             if (chdir(chrootRootDir.c_str()) == -1)
                 throw SysError("cannot change directory to '%1%'", chrootRootDir);

From ca2f64bcdaef5915f5147eac935ecb770511e438 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 15 May 2020 00:32:44 +0200
Subject: [PATCH 004/112] Reduce # of UIDs per build to 65536

2^18 was overkill. The idea was to enable multiple containers to run
inside a build. However, those containers can use the same UID range -
we don't really care about perfect isolation between containers inside
a build.
---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 7dc842bca..89db072b0 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -149,7 +149,7 @@ public:
         "The Unix group that contains the build users."};
 
     #if __linux__
-    const uint32_t idsPerBuild = 1 << 18;
+    const uint32_t idsPerBuild = 1 << 16;
 
     Setting<uint32_t> startId{this, 872415232, "start-id",
         "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"};

From 7bdcf43b401eba6aee29a359c5bce1f9cc01ce52 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 16 May 2020 21:09:48 +0200
Subject: [PATCH 005/112] Destroy the cgroup prior to building

---
 src/libstore/build.cc  |  8 +++----
 src/libstore/cgroup.cc | 49 ++++++++++++++++++++++++++++++++++++++++++
 src/libstore/cgroup.hh | 13 +++++++++++
 3 files changed, 66 insertions(+), 4 deletions(-)
 create mode 100644 src/libstore/cgroup.cc
 create mode 100644 src/libstore/cgroup.hh

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 816d695a5..1f1468d97 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -16,6 +16,7 @@
 #include "machines.hh"
 #include "daemon.hh"
 #include "worker-protocol.hh"
+#include "cgroup.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -2400,14 +2401,13 @@ void DerivationGoal::startBuilder()
             auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup);
 
             if (!pathExists(hostCgroup))
-                throw Error("expected unified cgroup directory '%s'", hostCgroup);
+                throw Error("expected cgroup directory '%s'", hostCgroup);
 
             auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID());
 
-            // FIXME: if the cgroup already exists, kill all processes
-            // in it and destroy it.
+            destroyCgroup(childCgroup);
 
-            if (mkdir(childCgroup.c_str(), 0755) == -1 && errno != EEXIST)
+            if (mkdir(childCgroup.c_str(), 0755) == -1)
                 throw SysError("creating cgroup '%s'", childCgroup);
 
             chownToBuilder(childCgroup);
diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc
new file mode 100644
index 000000000..8cd682e68
--- /dev/null
+++ b/src/libstore/cgroup.cc
@@ -0,0 +1,49 @@
+#if __linux__
+
+#include "cgroup.hh"
+#include "util.hh"
+
+#include <chrono>
+
+#include <dirent.h>
+
+namespace nix {
+
+void destroyCgroup(const Path & cgroup)
+{
+    for (auto & entry : readDirectory(cgroup)) {
+        if (entry.type != DT_DIR) continue;
+        destroyCgroup(cgroup + "/" + entry.name);
+    }
+
+    int round = 1;
+
+    while (true) {
+        auto pids = tokenizeString<std::vector<std::string>>(readFile(cgroup + "/cgroup.procs"));
+
+        if (pids.empty()) break;
+
+        if (round > 20)
+            throw Error("cannot kill cgroup '%s'", cgroup);
+
+        for (auto & pid_s : pids) {
+            pid_t pid;
+            if (!string2Int(pid_s, pid)) throw Error("invalid pid '%s'", pid);
+            // FIXME: pid wraparound
+            if (kill(pid, SIGKILL) == -1 && errno != ESRCH)
+                throw SysError("killing member %d of cgroup '%s'", pid, cgroup);
+        }
+
+        auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10)));
+        printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup);
+        std::this_thread::sleep_for(sleep);
+        round++;
+    }
+
+    if (rmdir(cgroup.c_str()) == -1)
+        throw SysError("deleting cgroup '%s'", cgroup);
+}
+
+}
+
+#endif
diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh
new file mode 100644
index 000000000..c7b09398e
--- /dev/null
+++ b/src/libstore/cgroup.hh
@@ -0,0 +1,13 @@
+#pragma once
+
+#if __linux__
+
+#include "types.hh"
+
+namespace nix {
+
+void destroyCgroup(const Path & cgroup);
+
+}
+
+#endif

From 570c443f560e015cf02e4f96102eaaa0e6853562 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 16 May 2020 21:21:41 +0200
Subject: [PATCH 006/112] Simplify cgroup creation

---
 src/libstore/build.cc  | 53 +++++++++++++-----------------------------
 src/libstore/cgroup.cc | 17 ++++++++++++++
 src/libstore/cgroup.hh |  2 ++
 3 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1f1468d97..97554e9cf 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2375,50 +2375,30 @@ void DerivationGoal::startBuilder()
 
 #if __linux__
     if (useChroot) {
-        /* Create a cgroup. */
+        /* Create a systemd cgroup since that's the minimum required
+           by systemd-nspawn. */
         // FIXME: do we want to use the parent cgroup? We should
         // always use the same cgroup regardless of whether we're the
         // daemon or run from a user session via sudo.
-        std::string msg;
-        std::vector<Path> cgroups;
-        for (auto & line : tokenizeString<std::vector<std::string>>(readFile("/proc/self/cgroup"), "\n")) {
-            static std::regex regex("([0-9]+):([^:]*):(.*)");
-            std::smatch match;
-            if (!std::regex_match(line, match, regex))
-                throw Error("invalid line '%s' in '/proc/self/cgroup'", line);
+        auto ourCgroups = getCgroups("/proc/self/cgroup");
+        auto systemdCgroup = ourCgroups["systemd"];
+        if (systemdCgroup == "")
+            throw Error("'systemd' cgroup does not exist");
 
-            /* We only create a systemd cgroup, since that's enough
-               for running systemd-nspawn. */
-            std::string name;
-            if (match[2] == "name=systemd")
-                name = "systemd";
-            //else if (match[2] == "")
-            //    name = "unified";
-            else continue;
+        auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup);
 
-            std::string cgroup = match[3];
+        if (!pathExists(hostCgroup))
+            throw Error("expected cgroup directory '%s'", hostCgroup);
 
-            auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup);
+        auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID());
 
-            if (!pathExists(hostCgroup))
-                throw Error("expected cgroup directory '%s'", hostCgroup);
+        destroyCgroup(childCgroup);
 
-            auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID());
+        if (mkdir(childCgroup.c_str(), 0755) == -1)
+            throw SysError("creating cgroup '%s'", childCgroup);
 
-            destroyCgroup(childCgroup);
-
-            if (mkdir(childCgroup.c_str(), 0755) == -1)
-                throw SysError("creating cgroup '%s'", childCgroup);
-
-            chownToBuilder(childCgroup);
-            chownToBuilder(childCgroup + "/cgroup.procs");
-            if (name == "unified") {
-                chownToBuilder(childCgroup + "/cgroup.threads");
-                chownToBuilder(childCgroup + "/cgroup.subtree_control");
-            }
-
-            cgroups.push_back(childCgroup);
-        }
+        chownToBuilder(childCgroup);
+        chownToBuilder(childCgroup + "/cgroup.procs");
 
         /* Set up private namespaces for the build:
 
@@ -2545,8 +2525,7 @@ void DerivationGoal::startBuilder()
             throw SysError("getting sandbox mount namespace");
 
         /* Move the child into its own cgroup. */
-        for (auto & childCgroup : cgroups)
-            writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
+        writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
 
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc
index 8cd682e68..887facdca 100644
--- a/src/libstore/cgroup.cc
+++ b/src/libstore/cgroup.cc
@@ -9,6 +9,23 @@
 
 namespace nix {
 
+std::map<std::string, std::string> getCgroups(const Path & cgroupFile)
+{
+    std::map<std::string, std::string> cgroups;
+
+    for (auto & line : tokenizeString<std::vector<std::string>>(readFile(cgroupFile), "\n")) {
+        static std::regex regex("([0-9]+):([^:]*):(.*)");
+        std::smatch match;
+        if (!std::regex_match(line, match, regex))
+            throw Error("invalid line '%s' in '%s'", line, cgroupFile);
+
+        std::string name = hasPrefix(match[2], "name=") ? std::string(match[2], 5) : match[2];
+        cgroups.insert_or_assign(name, match[3]);
+    }
+
+    return cgroups;
+}
+
 void destroyCgroup(const Path & cgroup)
 {
     for (auto & entry : readDirectory(cgroup)) {
diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh
index c7b09398e..dc6758957 100644
--- a/src/libstore/cgroup.hh
+++ b/src/libstore/cgroup.hh
@@ -6,6 +6,8 @@
 
 namespace nix {
 
+std::map<std::string, std::string> getCgroups(const Path & cgroupFile);
+
 void destroyCgroup(const Path & cgroup);
 
 }

From ba50c3efa3b2394f5a8372939bc600008cd25e7e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 19 May 2020 23:25:44 +0200
Subject: [PATCH 007/112] Add "uid-range" and "systemd-cgroup" system features

"uid-range" provides 65536 UIDs to a build and runs the build as root
in its user namespace. "systemd-cgroup" allows the build to mount the
systemd cgroup controller (needed for running systemd-nspawn and NixOS
containers).

Also, add a configuration option "auto-allocate-uids" which is needed
to enable these features, and some experimental feature gates.

So to enable support for containers you need the following in
nix.conf:

  experimental-features = auto-allocate-uids systemd-cgroup
  auto-allocate-uids = true
  system-features = uid-range systemd-cgroup
---
 src/libstore/build.cc     | 270 ++++++++++----------------------------
 src/libstore/cgroup.cc    |  19 ++-
 src/libstore/globals.hh   |   5 +-
 src/libstore/user-lock.cc | 212 ++++++++++++++++++++++++++++++
 src/libstore/user-lock.hh |  39 ++++++
 5 files changed, 340 insertions(+), 205 deletions(-)
 create mode 100644 src/libstore/user-lock.cc
 create mode 100644 src/libstore/user-lock.hh

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 97554e9cf..1f79a8d2d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -16,7 +16,7 @@
 #include "machines.hh"
 #include "daemon.hh"
 #include "worker-protocol.hh"
-#include "cgroup.hh"
+#include "user-lock.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -504,154 +504,6 @@ void handleDiffHook(
     }
 }
 
-//////////////////////////////////////////////////////////////////////
-
-
-class UserLock
-{
-private:
-    Path fnUserLock;
-    AutoCloseFD fdUserLock;
-
-    bool isEnabled = false;
-    uid_t uid = 0;
-    gid_t gid = 0;
-    std::vector<gid_t> supplementaryGIDs;
-
-public:
-    UserLock();
-
-    void kill();
-
-    uid_t getUID() { assert(uid); return uid; }
-    gid_t getGID() { assert(gid); return gid; }
-    uint32_t getIDCount() { return settings.idsPerBuild; }
-    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
-
-    bool findFreeUser();
-
-    bool enabled() { return isEnabled; }
-
-};
-
-
-UserLock::UserLock()
-{
-#if 0
-    assert(settings.buildUsersGroup != "");
-    createDirs(settings.nixStateDir + "/userpool");
-#endif
-}
-
-bool UserLock::findFreeUser() {
-    if (enabled()) return true;
-
-#if 0
-    /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
-    if (!gr)
-        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
-            settings.buildUsersGroup);
-    gid = gr->gr_gid;
-
-    /* Copy the result of getgrnam. */
-    Strings users;
-    for (char * * p = gr->gr_mem; *p; ++p) {
-        debug("found build user '%1%'", *p);
-        users.push_back(*p);
-    }
-
-    if (users.empty())
-        throw Error("the build users group '%1%' has no members",
-            settings.buildUsersGroup);
-
-    /* Find a user account that isn't currently in use for another
-       build. */
-    for (auto & i : users) {
-        debug("trying user '%1%'", i);
-
-        struct passwd * pw = getpwnam(i.c_str());
-        if (!pw)
-            throw Error("the user '%1%' in the group '%2%' does not exist",
-                i, settings.buildUsersGroup);
-
-
-        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            user = i;
-            uid = pw->pw_uid;
-
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error("the Nix user should not be a member of '%1%'",
-                    settings.buildUsersGroup);
-
-#if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
-#endif
-
-            isEnabled = true;
-            return true;
-        }
-    }
-
-    return false;
-#endif
-
-    assert(settings.startId > 0);
-    assert(settings.startId % settings.idsPerBuild == 0);
-    assert(settings.uidCount % settings.idsPerBuild == 0);
-    assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
-
-    // FIXME: check whether the id range overlaps any known users
-
-    size_t nrSlots = settings.uidCount / settings.idsPerBuild;
-
-    for (size_t i = 0; i < nrSlots; i++) {
-        debug("trying user slot '%d'", i);
-
-        createDirs(settings.nixStateDir + "/userpool");
-
-        fnUserLock = fmt("%s/userpool/slot-%d", settings.nixStateDir, i);
-
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError("opening user lock '%1%'", fnUserLock);
-
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            uid = settings.startId + i * settings.idsPerBuild;
-            gid = settings.startId + i * settings.idsPerBuild;
-            return true;
-        }
-    }
-
-    return false;
-}
-
-void UserLock::kill()
-{
-    // FIXME: use a cgroup to kill all processes in the build?
-#if 0
-    killUser(uid);
-#endif
-}
-
 
 //////////////////////////////////////////////////////////////////////
 
@@ -840,6 +692,13 @@ private:
 
     Path chrootRootDir;
 
+    /* Whether to give the build more than 1 UID. */
+    bool useUidRange = false;
+
+    /* Whether to make the 'systemd' cgroup controller available to
+       the build. */
+    bool useSystemdCgroup = false;
+
     /* RAII object to delete the chroot directory. */
     std::shared_ptr<AutoDelete> autoDelChroot;
 
@@ -896,8 +755,8 @@ private:
        result. */
     std::map<Path, ValidPathInfo> prevInfos;
 
-    const uid_t sandboxUid = 1000;
-    const gid_t sandboxGid = 100;
+    uid_t sandboxUid = -1;
+    gid_t sandboxGid = -1;
 
     const static Path homeDir;
 
@@ -1445,6 +1304,7 @@ void DerivationGoal::inputsRealised()
     result = BuildResult();
 }
 
+
 void DerivationGoal::started() {
     auto msg = fmt(
         buildMode == bmRepair ? "repairing outputs of '%s'" :
@@ -1459,6 +1319,7 @@ void DerivationGoal::started() {
     worker.updateProgress();
 }
 
+
 void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
@@ -1556,25 +1417,28 @@ void DerivationGoal::tryToBuild()
     worker.wakeUp(shared_from_this());
 }
 
+
 void DerivationGoal::tryLocalBuild() {
 
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
-    if ((settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0) {
+    static bool useBuildUsers = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0;
+    if (useBuildUsers) {
 #if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
+        if (!buildUser)
+            buildUser = acquireUserLock();
 
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
+        if (!buildUser) {
             if (!actLock)
                 actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
                     fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
             worker.waitForAWhile(shared_from_this());
             return;
         }
+
+        /* Make sure that no other processes are executing under this
+           uid. */
+        buildUser->kill();
 #else
         /* Don't know how to block the creation of setuid/setgid
            binaries on this platform. */
@@ -2087,6 +1951,9 @@ void DerivationGoal::startBuilder()
         }
     }
 
+    useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range");
+    useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("systemd-cgroup");
+
     if (useChroot) {
 
         /* Allow a user-configurable set of directories from the
@@ -2166,7 +2033,7 @@ void DerivationGoal::startBuilder()
 
         printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
 
-        if (mkdir(chrootRootDir.c_str(), 0755) == -1)
+        if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
         // FIXME: only make root writable for user namespace builds.
@@ -2186,6 +2053,12 @@ void DerivationGoal::startBuilder()
         createDirs(chrootRootDir + "/etc");
         chownToBuilder(chrootRootDir + "/etc");
 
+        if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536))
+            throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name);
+
+        sandboxUid = useUidRange ? 0 : 1000;
+        sandboxGid = useUidRange ? 0 : 100;
+
         writeFile(chrootRootDir + "/etc/passwd", fmt(
                 "root:x:0:0:Nix build user:%3%:/noshell\n"
                 "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
@@ -2238,12 +2111,32 @@ void DerivationGoal::startBuilder()
         for (auto & i : drv->outputs)
             dirsInChroot.erase(worker.store.printStorePath(i.second.path));
 
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
+        if (useSystemdCgroup) {
+            settings.requireExperimentalFeature("systemd-cgroup");
+            std::optional<Path> cgroup;
+            if (!buildUser || !(cgroup = buildUser->getCgroup()))
+                throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf");
+            chownToBuilder(*cgroup);
+            chownToBuilder(*cgroup + "/cgroup.procs");
+        }
+
 #else
-        throw Error("sandboxing builds is not supported on this platform");
+        if (useUidRange)
+            throw Error("feature 'uid-range' is not supported on this platform");
+        if (useSystemdCgroup)
+            throw Error("feature 'systemd-cgroup' is not supported on this platform");
+        #if __APPLE__
+            /* We don't really have any parent prep work to do (yet?)
+               All work happens in the child, instead. */
+        #else
+            throw Error("sandboxing builds is not supported on this platform");
+        #endif
 #endif
+    } else {
+        if (useUidRange)
+            throw Error("feature 'uid-range' is only supported in sandboxed builds");
+        if (useSystemdCgroup)
+            throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds");
     }
 
     if (needsHashRewrite()) {
@@ -2375,31 +2268,6 @@ void DerivationGoal::startBuilder()
 
 #if __linux__
     if (useChroot) {
-        /* Create a systemd cgroup since that's the minimum required
-           by systemd-nspawn. */
-        // FIXME: do we want to use the parent cgroup? We should
-        // always use the same cgroup regardless of whether we're the
-        // daemon or run from a user session via sudo.
-        auto ourCgroups = getCgroups("/proc/self/cgroup");
-        auto systemdCgroup = ourCgroups["systemd"];
-        if (systemdCgroup == "")
-            throw Error("'systemd' cgroup does not exist");
-
-        auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup);
-
-        if (!pathExists(hostCgroup))
-            throw Error("expected cgroup directory '%s'", hostCgroup);
-
-        auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID());
-
-        destroyCgroup(childCgroup);
-
-        if (mkdir(childCgroup.c_str(), 0755) == -1)
-            throw SysError("creating cgroup '%s'", childCgroup);
-
-        chownToBuilder(childCgroup);
-        chownToBuilder(childCgroup + "/cgroup.procs");
-
         /* Set up private namespaces for the build:
 
            - The PID namespace causes the build to start as PID 1.
@@ -2508,15 +2376,16 @@ void DerivationGoal::startBuilder()
            the calling user (if build users are disabled). */
         uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
         uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-        uint32_t nrIds = settings.idsPerBuild; // FIXME
+        uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1;
 
         writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-            fmt("%d %d %d", /* sandboxUid */ 0, hostUid, nrIds));
+            fmt("%d %d %d", sandboxUid, hostUid, nrIds));
 
-        //writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+        if (!useUidRange)
+            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
 
         writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-            fmt("%d %d %d", /* sandboxGid */ 0, hostGid, nrIds));
+            fmt("%d %d %d", sandboxGid, hostGid, nrIds));
 
         /* Save the mount namespace of the child. We have to do this
            *before* the child does a chroot. */
@@ -2525,7 +2394,10 @@ void DerivationGoal::startBuilder()
             throw SysError("getting sandbox mount namespace");
 
         /* Move the child into its own cgroup. */
-        writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
+        if (buildUser) {
+            if (auto cgroup = buildUser->getCgroup())
+                writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
+        }
 
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
@@ -3361,7 +3233,7 @@ void DerivationGoal::runChild()
             /* Unshare the cgroup namespace. This means
                /proc/self/cgroup will show the child's cgroup as '/'
                rather than whatever it is in the parent. */
-            if (unshare(CLONE_NEWCGROUP) == -1)
+            if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1)
                 throw SysError("unsharing cgroup namespace");
 
             /* Do the chroot(). */
@@ -3386,16 +3258,10 @@ void DerivationGoal::runChild()
             /* Switch to the sandbox uid/gid in the user namespace,
                which corresponds to the build user or calling user in
                the parent namespace. */
-#if 0
             if (setgid(sandboxGid) == -1)
                 throw SysError("setgid failed");
             if (setuid(sandboxUid) == -1)
                 throw SysError("setuid failed");
-#endif
-            if (setgid(0) == -1)
-                throw SysError("setgid failed");
-            if (setuid(0) == -1)
-                throw SysError("setuid failed");
 
             setUser = false;
         }
@@ -3789,7 +3655,7 @@ void DerivationGoal::registerOutputs()
                something like that. */
             canonicalisePathMetaData(
                 actualPath,
-                buildUser ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) : std::nullopt,
+                buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt,
                 inodesSeen);
 
             /* FIXME: this is in-memory. */
@@ -3866,7 +3732,7 @@ void DerivationGoal::registerOutputs()
            all files are owned by the build user, if applicable. */
         canonicalisePathMetaData(actualPath,
             buildUser && !rewritten
-            ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1))
+            ? std::optional(buildUser->getUIDRange())
             : std::nullopt,
             inodesSeen);
 
diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc
index 887facdca..9e5e937df 100644
--- a/src/libstore/cgroup.cc
+++ b/src/libstore/cgroup.cc
@@ -4,6 +4,7 @@
 #include "util.hh"
 
 #include <chrono>
+#include <unordered_set>
 
 #include <dirent.h>
 
@@ -19,7 +20,7 @@ std::map<std::string, std::string> getCgroups(const Path & cgroupFile)
         if (!std::regex_match(line, match, regex))
             throw Error("invalid line '%s' in '%s'", line, cgroupFile);
 
-        std::string name = hasPrefix(match[2], "name=") ? std::string(match[2], 5) : match[2];
+        std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2];
         cgroups.insert_or_assign(name, match[3]);
     }
 
@@ -28,6 +29,8 @@ std::map<std::string, std::string> getCgroups(const Path & cgroupFile)
 
 void destroyCgroup(const Path & cgroup)
 {
+    if (!pathExists(cgroup)) return;
+
     for (auto & entry : readDirectory(cgroup)) {
         if (entry.type != DT_DIR) continue;
         destroyCgroup(cgroup + "/" + entry.name);
@@ -35,6 +38,8 @@ void destroyCgroup(const Path & cgroup)
 
     int round = 1;
 
+    std::unordered_set<pid_t> pidsShown;
+
     while (true) {
         auto pids = tokenizeString<std::vector<std::string>>(readFile(cgroup + "/cgroup.procs"));
 
@@ -46,13 +51,23 @@ void destroyCgroup(const Path & cgroup)
         for (auto & pid_s : pids) {
             pid_t pid;
             if (!string2Int(pid_s, pid)) throw Error("invalid pid '%s'", pid);
+            if (pidsShown.insert(pid).second) {
+                try {
+                    auto cmdline = readFile(fmt("/proc/%d/cmdline", pid));
+                    using namespace std::string_literals;
+                    warn("killing stray builder process %d (%s)...",
+                        pid, trim(replaceStrings(cmdline, "\0"s, " ")));
+                } catch (SysError &) {
+                }
+            }
             // FIXME: pid wraparound
             if (kill(pid, SIGKILL) == -1 && errno != ESRCH)
                 throw SysError("killing member %d of cgroup '%s'", pid, cgroup);
         }
 
         auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10)));
-        printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup);
+        if (sleep.count() > 100)
+            printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup);
         std::this_thread::sleep_for(sleep);
         round++;
     }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 89db072b0..5cf73c7b4 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -149,10 +149,13 @@ public:
         "The Unix group that contains the build users."};
 
     #if __linux__
+    Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids",
+        "Whether to allocate UIDs for builders automatically."};
+
     const uint32_t idsPerBuild = 1 << 16;
 
     Setting<uint32_t> startId{this, 872415232, "start-id",
-        "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"};
+        "The first UID and GID to use for dynamic ID allocation."};
 
     Setting<uint32_t> uidCount{this, idsPerBuild * 128, "id-count",
         "The number of UIDs/GIDs to use for dynamic ID allocation."};
diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc
new file mode 100644
index 000000000..8a09df4d1
--- /dev/null
+++ b/src/libstore/user-lock.cc
@@ -0,0 +1,212 @@
+#include "user-lock.hh"
+#include "globals.hh"
+#include "pathlocks.hh"
+#include "cgroup.hh"
+
+namespace nix {
+
+struct SimpleUserLock : UserLock
+{
+    AutoCloseFD fdUserLock;
+    uid_t uid;
+    gid_t gid;
+    std::vector<gid_t> supplementaryGIDs;
+
+    void kill() override
+    {
+        killUser(uid);
+    }
+
+    std::pair<uid_t, uid_t> getUIDRange() override
+    {
+        assert(uid);
+        return {uid, uid};
+    }
+
+    gid_t getGID() override { assert(gid); return gid; }
+
+    std::vector<gid_t> getSupplementaryGIDs() override { return supplementaryGIDs; }
+
+    static std::unique_ptr<UserLock> acquire()
+    {
+        assert(settings.buildUsersGroup != "");
+        createDirs(settings.nixStateDir + "/userpool");
+
+        /* Get the members of the build-users-group. */
+        struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
+        if (!gr)
+            throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
+
+        /* Copy the result of getgrnam. */
+        Strings users;
+        for (char * * p = gr->gr_mem; *p; ++p) {
+            debug("found build user '%s'", *p);
+            users.push_back(*p);
+        }
+
+        if (users.empty())
+            throw Error("the build users group '%s' has no members", settings.buildUsersGroup);
+
+        /* Find a user account that isn't currently in use for another
+           build. */
+        for (auto & i : users) {
+            debug("trying user '%s'", i);
+
+            struct passwd * pw = getpwnam(i.c_str());
+            if (!pw)
+                throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup);
+
+            auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir,pw->pw_uid);
+
+            AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+            if (!fd)
+                throw SysError("opening user lock '%s'", fnUserLock);
+
+            if (lockFile(fd.get(), ltWrite, false)) {
+                auto lock = std::make_unique<SimpleUserLock>();
+
+                lock->fdUserLock = std::move(fd);
+                lock->uid = pw->pw_uid;
+                lock->gid = gr->gr_gid;
+
+                /* Sanity check... */
+                if (lock->uid == getuid() || lock->uid == geteuid())
+                    throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup);
+
+                #if __linux__
+                /* Get the list of supplementary groups of this build
+                   user.  This is usually either empty or contains a
+                   group such as "kvm".  */
+                lock->supplementaryGIDs.resize(10);
+                int ngroups = lock->supplementaryGIDs.size();
+                int err = getgrouplist(pw->pw_name, pw->pw_gid,
+                    lock->supplementaryGIDs.data(), &ngroups);
+                if (err == -1)
+                    throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name);
+
+                lock->supplementaryGIDs.resize(ngroups);
+                #endif
+
+                return lock;
+            }
+        }
+
+        return nullptr;
+    }
+};
+
+#if __linux__
+struct CgroupUserLock : UserLock
+{
+    AutoCloseFD fdUserLock;
+    uid_t uid;
+
+    void kill() override
+    {
+        if (cgroup) {
+            destroyCgroup(*cgroup);
+            cgroup.reset();
+        }
+    }
+
+    std::pair<uid_t, uid_t> getUIDRange() override
+    {
+        assert(uid);
+        return {uid, uid + settings.idsPerBuild - 1};
+    }
+
+    gid_t getGID() override
+    {
+        // We use the same GID ranges as for the UIDs.
+        assert(uid);
+        return uid;
+    }
+
+    std::vector<gid_t> getSupplementaryGIDs() override { return {}; } // FIXME
+
+    static std::unique_ptr<UserLock> acquire()
+    {
+        settings.requireExperimentalFeature("auto-allocate-uids");
+        assert(settings.startId > 0);
+        assert(settings.startId % settings.idsPerBuild == 0);
+        assert(settings.uidCount % settings.idsPerBuild == 0);
+        assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
+
+        // FIXME: check whether the id range overlaps any known users
+
+        createDirs(settings.nixStateDir + "/userpool2");
+
+        size_t nrSlots = settings.uidCount / settings.idsPerBuild;
+
+        for (size_t i = 0; i < nrSlots; i++) {
+            debug("trying user slot '%d'", i);
+
+            createDirs(settings.nixStateDir + "/userpool2");
+
+            auto fnUserLock = fmt("%s/userpool2/slot-%d", settings.nixStateDir, i);
+
+            AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+            if (!fd)
+                throw SysError("opening user lock '%s'", fnUserLock);
+
+            if (lockFile(fd.get(), ltWrite, false)) {
+                auto lock = std::make_unique<CgroupUserLock>();
+                lock->fdUserLock = std::move(fd);
+                lock->uid = settings.startId + i * settings.idsPerBuild;
+                auto s = drainFD(lock->fdUserLock.get());
+                if (s != "") lock->cgroup = s;
+                return lock;
+            }
+        }
+
+        return nullptr;
+    }
+
+    std::optional<Path> cgroup;
+
+    std::optional<Path> getCgroup() override
+    {
+        if (!cgroup) {
+            /* Create a systemd cgroup since that's the minimum
+               required by systemd-nspawn. */
+            auto ourCgroups = getCgroups("/proc/self/cgroup");
+            auto systemdCgroup = ourCgroups["systemd"];
+            if (systemdCgroup == "")
+                throw Error("'systemd' cgroup does not exist");
+
+            auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup);
+
+            if (!pathExists(hostCgroup))
+                throw Error("expected cgroup directory '%s'", hostCgroup);
+
+            cgroup = fmt("%s/nix-%d", hostCgroup, uid);
+
+            destroyCgroup(*cgroup);
+
+            if (mkdir(cgroup->c_str(), 0755) == -1)
+                throw SysError("creating cgroup '%s'", *cgroup);
+
+            /* Record the cgroup in the lock file. This ensures that
+               if we subsequently get executed under a different parent
+               cgroup, we kill the previous cgroup first. */
+            if (ftruncate(fdUserLock.get(), 0) == -1)
+                throw Error("truncating user lock");
+            writeFull(fdUserLock.get(), *cgroup);
+        }
+
+        return cgroup;
+    };
+};
+#endif
+
+std::unique_ptr<UserLock> acquireUserLock()
+{
+    #if __linux__
+    if (settings.autoAllocateUids)
+        return CgroupUserLock::acquire();
+    else
+    #endif
+        return SimpleUserLock::acquire();
+}
+
+}
diff --git a/src/libstore/user-lock.hh b/src/libstore/user-lock.hh
new file mode 100644
index 000000000..88d068689
--- /dev/null
+++ b/src/libstore/user-lock.hh
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+struct UserLock
+{
+    virtual ~UserLock() { }
+
+    /* Get the first and last UID. */
+    virtual std::pair<uid_t, uid_t> getUIDRange() = 0;
+
+    /* Get the first UID. */
+    uid_t getUID()
+    {
+        return getUIDRange().first;
+    }
+
+    uid_t getUIDCount()
+    {
+        return getUIDRange().second - getUIDRange().first + 1;
+    }
+
+    virtual gid_t getGID() = 0;
+
+    virtual std::vector<gid_t> getSupplementaryGIDs() = 0;
+
+    /* Kill any processes currently executing as this user. */
+    virtual void kill() = 0;
+
+    virtual std::optional<Path> getCgroup() { return {}; };
+};
+
+/* Acquire a user lock. Note that this may return nullptr if no user
+   is available. */
+std::unique_ptr<UserLock> acquireUserLock();
+
+}

From 8c4cce553c16438f0ccbbaea6d77f2bd64306dfe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 May 2020 11:24:21 +0200
Subject: [PATCH 008/112] Fix macOS build

---
 src/libstore/build.cc     | 11 +----------
 src/libstore/user-lock.cc | 13 +++++++++++++
 src/libstore/user-lock.hh |  2 ++
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1f79a8d2d..6c3f94a76 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1420,11 +1420,7 @@ void DerivationGoal::tryToBuild()
 
 void DerivationGoal::tryLocalBuild() {
 
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    static bool useBuildUsers = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0;
-    if (useBuildUsers) {
-#if defined(__linux__) || defined(__APPLE__)
+    if (useBuildUsers()) {
         if (!buildUser)
             buildUser = acquireUserLock();
 
@@ -1439,11 +1435,6 @@ void DerivationGoal::tryLocalBuild() {
         /* Make sure that no other processes are executing under this
            uid. */
         buildUser->kill();
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
     }
 
     actLock.reset();
diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc
index 8a09df4d1..2254386da 100644
--- a/src/libstore/user-lock.cc
+++ b/src/libstore/user-lock.cc
@@ -209,4 +209,17 @@ std::unique_ptr<UserLock> acquireUserLock()
         return SimpleUserLock::acquire();
 }
 
+bool useBuildUsers()
+{
+    #if __linux__
+    static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0;
+    return b;
+    #elif __APPLE__
+    static bool b = settings.buildUsersGroup != "" && getuid() == 0;
+    return b;
+    #else
+    return false;
+    #endif
+}
+
 }
diff --git a/src/libstore/user-lock.hh b/src/libstore/user-lock.hh
index 88d068689..bfb55b0d9 100644
--- a/src/libstore/user-lock.hh
+++ b/src/libstore/user-lock.hh
@@ -36,4 +36,6 @@ struct UserLock
    is available. */
 std::unique_ptr<UserLock> acquireUserLock();
 
+bool useBuildUsers();
+
 }

From 7349f257da8278af9aae35544b15c9a204e2a57b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 20 May 2020 11:57:33 +0200
Subject: [PATCH 009/112] Only mount /sys in uid-range builds

Maybe this should be a separate system feature... /sys exposes a lot
of impure info about the host system.
---
 src/libstore/build.cc     | 11 ++++++-----
 src/libstore/user-lock.cc |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6c3f94a76..e927a65f0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3173,11 +3173,12 @@ void DerivationGoal::runChild()
             if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
                 throw SysError("mounting /proc");
 
-            /* Mount sysfs on /sys. FIXME: only in user namespace
-               builds. */
-            createDirs(chrootRootDir + "/sys");
-            if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
-                throw SysError("mounting /sys");
+            /* Mount sysfs on /sys. */
+            if (useUidRange) {
+                createDirs(chrootRootDir + "/sys");
+                if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
+                    throw SysError("mounting /sys");
+            }
 
             /* Mount a new tmpfs on /dev/shm to ensure that whatever
                the builder puts in /dev/shm is cleaned up automatically. */
diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc
index 2254386da..fb2a45f48 100644
--- a/src/libstore/user-lock.cc
+++ b/src/libstore/user-lock.cc
@@ -122,7 +122,7 @@ struct CgroupUserLock : UserLock
         return uid;
     }
 
-    std::vector<gid_t> getSupplementaryGIDs() override { return {}; } // FIXME
+    std::vector<gid_t> getSupplementaryGIDs() override { return {}; }
 
     static std::unique_ptr<UserLock> acquire()
     {

From 49119072e72ae160c22a2c0b963cfe732b2819e4 Mon Sep 17 00:00:00 2001
From: Adam Joseph <adam@westernsemico.com>
Date: Wed, 27 Apr 2022 17:55:04 -0700
Subject: [PATCH 010/112] local-derivation-goal.cc: seccomp filters for MIPS
 secondary arch/abi

A mips64el Linux MIPS kernel can execute userspace code using any of
three ABIs:

  mips64el-linux-*abin64
  mips64el-linux-*abin32
    mipsel-linux-*

The first of these is the native 64-bit ABI, and the only ABI with
64-bit pointers; this is sometimes called "n64".  The last of these is
the old legacy 32-bit ABI, whose binaries can execute natively on
32-bit MIPS hardware; this is sometimes called "o32".

The second ABI, "n32" is essentially the 64-bit ABI with 32-bit
pointers and address space.  Hardware 64-bit integer/floating
arithmetic is still allowed, as well as the much larger mips64
register set and more-efficient calling convention.

Let's enable seccomp filters for all of these.  Likewise for big
endian (mips64-linux-*).
---
 src/libstore/build/local-derivation-goal.cc | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 4c91fa4fb..f80a4678b 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1529,6 +1529,22 @@ void setupSeccomp()
         seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
         printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
 
+    if (nativeSystem == "mips64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0)
+        printError("unable to add mips seccomp architecture");
+
+    if (nativeSystem == "mips64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0)
+        printError("unable to add mips64-*abin32 seccomp architecture");
+
+    if (nativeSystem == "mips64el-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0)
+        printError("unable to add mipsel seccomp architecture");
+
+    if (nativeSystem == "mips64el-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0)
+        printError("unable to add mips64el-*abin32 seccomp architecture");
+
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,

From 3378a3bce8a5dc1282d901f7f3ec152676198092 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:07:18 +0200
Subject: [PATCH 011/112] add syntax overview from NixOS manual

taken verbatim to keep track of required corrections.

made it an HTML table to more easily change structure and keep diffs
minimal.
---
 doc/manual/src/language/index.md | 306 +++++++++++++++++++++++++++++++
 1 file changed, 306 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index c4b3abf75..6c355d923 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -11,3 +11,309 @@ packages, compositions of packages, and the variability within packages.
 
 This section presents the various features of the language.
 
+# Syntax Summary
+
+Below is a summary of the most important syntactic constructs in the Nix
+expression language. It's not complete. In particular, there are many
+other built-in functions. See the [Nix
+manual](https://nixos.org/nix/manual/#chap-writing-nix-expressions) for
+the rest.
+
+<table>
+  <tr>
+    <th>
+       Example
+    </th>
+    <th>
+      Description
+    </th>
+  </tr>
+  <tr>
+    <td>
+       *Basic values*
+    </td>
+    <td>
+
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `"Hello world"`
+    </td>
+    <td>
+      A string
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `"${pkgs.bash}/bin/sh"`
+    </td>
+    <td>
+      A string containing an expression (expands to `"/nix/store/hash-bash-version/bin/sh"`)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `true`, `false`
+    </td>
+    <td>
+      Booleans
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `123`
+    </td>
+    <td>
+      An integer
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `./foo.png`
+    </td>
+    <td>
+      A path (relative to the containing Nix expression)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       *Compound values*
+    </td>
+    <td>
+
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x = 1; y = 2; }`
+    </td>
+    <td>
+      A set with attributes named `x` and `y`
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ foo.bar = 1; }`
+    </td>
+    <td>
+      A nested set, equivalent to `{ foo = { bar = 1; }; }`
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `rec { x = "foo"; y = x + "bar"; }`
+    </td>
+    <td>
+      A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }`
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `[ "foo" "bar" ]`
+    </td>
+    <td>
+      A list with two elements
+    </td>
+  </tr>
+  <tr>
+    <td>
+       *Operators*
+    </td>
+    <td>
+
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `"foo" + "bar"`
+    </td>
+    <td>
+      String concatenation
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `1 + 2`
+    </td>
+    <td>
+      Integer addition
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `"foo" == "f" + "oo"`
+    </td>
+    <td>
+      Equality test (evaluates to `true`)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `"foo" != "bar"`
+    </td>
+    <td>
+      Inequality test (evaluates to `true`)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `!true`
+    </td>
+    <td>
+      Boolean negation
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x = 1; y = 2; }.x`
+    </td>
+    <td>
+      Attribute selection (evaluates to `1`)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x = 1; y = 2; }.z or 3`
+    </td>
+    <td>
+      Attribute selection with default (evaluates to `3`)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x = 1; y = 2; } // { z = 3; }`
+    </td>
+    <td>
+      Merge two sets (attributes in the right-hand set taking precedence)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       *Control structures*
+    </td>
+    <td>
+
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `if 1 + 1 == 2 then "yes!" else "no!"`
+    </td>
+    <td>
+      Conditional expression
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `assert 1 + 1 == 2; "yes!"`
+    </td>
+    <td>
+      Assertion check (evaluates to `"yes!"`). See [](#sec-assertions) for using assertions in modules
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `let x = "foo"; y = "bar"; in x + y`
+    </td>
+    <td>
+      Variable definition
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `with pkgs.lib; head [ 1 2 3 ]`
+    </td>
+    <td>
+      Add all attributes from the given set to the scope (evaluates to `1`)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       *Functions (lambdas)*
+    </td>
+    <td>
+
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `x: x + 1`
+    </td>
+    <td>
+      A function that expects an integer and returns it increased by 1
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `(x: x + 1) 100`
+    </td>
+    <td>
+      A function call (evaluates to 101)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `let inc = x: x + 1; in inc (inc (inc 100))`
+    </td>
+    <td>
+      A function bound to a variable and subsequently called by name (evaluates to 103)
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x, y }: x + y`
+    </td>
+    <td>
+      A function that expects a set with required attributes `x` and `y` and concatenates them
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x, y ? "bar" }: x + y`
+    </td>
+    <td>
+      A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y`
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x, y, ... }: x + y`
+    </td>
+    <td>
+      A function that expects a set with required attributes `x` and `y` and ignores any other attributes
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `{ x, y } @ args: x + y`
+    </td>
+    <td>
+      A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args`
+    </td>
+  </tr>
+  <tr>
+    <td>
+       *Built-in functions*
+    </td>
+    <td>
+
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `import ./foo.nix`
+    </td>
+    <td>
+      Load and return Nix expression in given file
+    </td>
+  </tr>
+  <tr>
+    <td>
+       `map (x: x + x) [ 1 2 3 ]`
+    </td>
+    <td>
+      Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`)
+    </td>
+  </tr>
+</table>

From 90836397d36080b097740fee587839e458763cf9 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:09:45 +0200
Subject: [PATCH 012/112] remove stale section link

---
 doc/manual/src/language/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 6c355d923..f22d66351 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -209,7 +209,7 @@ the rest.
        `assert 1 + 1 == 2; "yes!"`
     </td>
     <td>
-      Assertion check (evaluates to `"yes!"`). See [](#sec-assertions) for using assertions in modules
+      Assertion check (evaluates to `"yes!"`).
     </td>
   </tr>
   <tr>

From 43188d3d1818214fba6037af7ea3e87df5dfbdce Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:10:15 +0200
Subject: [PATCH 013/112] make hash and version distinguishable as placeholder

---
 doc/manual/src/language/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index f22d66351..41d20d835 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -49,7 +49,7 @@ the rest.
        `"${pkgs.bash}/bin/sh"`
     </td>
     <td>
-      A string containing an expression (expands to `"/nix/store/hash-bash-version/bin/sh"`)
+      A string containing an expression (expands to `"/nix/store/<hash>-bash-<version>/bin/sh"`)
     </td>
   </tr>
   <tr>

From f165a8ae0874529f979a2bf33283300bb1ccdef4 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:10:48 +0200
Subject: [PATCH 014/112] flarify relative path semantics

---
 doc/manual/src/language/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 41d20d835..f601d1e16 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -73,7 +73,7 @@ the rest.
        `./foo.png`
     </td>
     <td>
-      A path (relative to the containing Nix expression)
+      A path relative to the file containing this Nix expression
     </td>
   </tr>
   <tr>

From 292cab039d7c2d1dc2e67f74a92dbf4953430935 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:16:13 +0200
Subject: [PATCH 015/112] add multi-line string

---
 doc/manual/src/language/index.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index f601d1e16..8241cd0cd 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -43,6 +43,18 @@ the rest.
     <td>
       A string
     </td>
+    <td>
+       ```
+        ''
+          multi
+           line
+            string
+        ''
+       ```
+    </td>
+    <td>
+      A multiline string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`.
+    </td>
   </tr>
   <tr>
     <td>

From e6f7c180de3553dbb4ebd3969f5da4423dafc4bd Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:18:59 +0200
Subject: [PATCH 016/112] add floating point number

---
 doc/manual/src/language/index.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 8241cd0cd..e623bc540 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -53,7 +53,7 @@ the rest.
        ```
     </td>
     <td>
-      A multiline string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`.
+      A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`.
     </td>
   </tr>
   <tr>
@@ -80,6 +80,14 @@ the rest.
       An integer
     </td>
   </tr>
+  <tr>
+    <td>
+       `3.141`
+    </td>
+    <td>
+      A floating point number
+    </td>
+  </tr>
   <tr>
     <td>
        `./foo.png`

From 21438acc70575134e6d17e280f5172dc2bec971f Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:20:32 +0200
Subject: [PATCH 017/112] add absolute path

---
 doc/manual/src/language/index.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index e623bc540..b4ffe171b 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -88,6 +88,14 @@ the rest.
       A floating point number
     </td>
   </tr>
+  <tr>
+    <td>
+       `/etc`
+    </td>
+    <td>
+      An absolute path
+    </td>
+  </tr>
   <tr>
     <td>
        `./foo.png`

From 587ae9ada5e1a94184772542352df85549f0ce99 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:22:41 +0200
Subject: [PATCH 018/112] add search path

---
 doc/manual/src/language/index.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index b4ffe171b..3ed00d8e3 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -104,6 +104,14 @@ the rest.
       A path relative to the file containing this Nix expression
     </td>
   </tr>
+  <tr>
+    <td>
+       <nixpkgs>
+    </td>
+    <td>
+      Search path. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH).
+    </td>
+  </tr>
   <tr>
     <td>
        *Compound values*

From 391fd10b12b00a3f61b178811fdecda7308e5193 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:25:40 +0200
Subject: [PATCH 019/112] add home path

---
 doc/manual/src/language/index.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 3ed00d8e3..5ad235819 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -104,6 +104,14 @@ the rest.
       A path relative to the file containing this Nix expression
     </td>
   </tr>
+  <tr>
+    <td>
+       `~/.config`
+    </td>
+    <td>
+      A home path. Evaluates to the `"<user's home directory>/.config"`.
+    </td>
+  </tr>
   <tr>
     <td>
        <nixpkgs>

From 5c25bdee509af297caddd7b6fdca275940282269 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 13:26:11 +0200
Subject: [PATCH 020/112] add null

---
 doc/manual/src/language/index.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 5ad235819..bc20f4316 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -72,6 +72,14 @@ the rest.
       Booleans
     </td>
   </tr>
+  <tr>
+    <td>
+       `null`
+    </td>
+    <td>
+      Null value
+    </td>
+  </tr>
   <tr>
     <td>
        `123`

From 2e4704ca9372c90826022d52360b6d57b564de5b Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 14:13:03 +0200
Subject: [PATCH 021/112] add second @ pattern example

---
 doc/manual/src/language/index.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index bc20f4316..f201bbded 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -339,6 +339,8 @@ the rest.
   <tr>
     <td>
        `{ x, y } @ args: x + y`
+
+       `args @ { x, y }: x + y`
     </td>
     <td>
       A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args`

From 0378531bf2cdb19ab4601c6dbc04c3f2cd962caa Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 14:13:15 +0200
Subject: [PATCH 022/112] add curried function

---
 doc/manual/src/language/index.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index f201bbded..72e09b25d 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -296,6 +296,14 @@ the rest.
       A function that expects an integer and returns it increased by 1
     </td>
   </tr>
+  <tr>
+    <td>
+       `x: y: x + y`
+    </td>
+    <td>
+      Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum.
+    </td>
+  </tr>
   <tr>
     <td>
        `(x: x + 1) 100`

From c209e6e108c95ba657992b208ce12af665eed9d4 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 14:13:36 +0200
Subject: [PATCH 023/112] add more list examples

---
 doc/manual/src/language/index.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 72e09b25d..65463baa0 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -162,10 +162,14 @@ the rest.
   </tr>
   <tr>
     <td>
-       `[ "foo" "bar" ]`
+       `[ "foo" "bar" "baz" ]`
+
+       `[ 1 2 3 ]`
+
+       `[ (f 1) { a = 1; b = 2; } [ "c" ] ]`
     </td>
     <td>
-      A list with two elements
+      Lists with three elements.
     </td>
   </tr>
   <tr>

From 6ba8d6dc82f2b44734f5760225435642ae842725 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 12 Aug 2022 14:27:03 +0200
Subject: [PATCH 024/112] add more examples on string interpolation

---
 doc/manual/src/language/index.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 65463baa0..724484460 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -38,7 +38,7 @@ the rest.
   </tr>
   <tr>
     <td>
-       `"Hello world"`
+       `"hello world"`
     </td>
     <td>
       A string
@@ -58,10 +58,14 @@ the rest.
   </tr>
   <tr>
     <td>
+       `"hello ${ { a = "world" }.a }"`
+
+       `"1 2 ${3}"`
+
        `"${pkgs.bash}/bin/sh"`
     </td>
     <td>
-      A string containing an expression (expands to `"/nix/store/<hash>-bash-<version>/bin/sh"`)
+      String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/<hash>-bash-<version>/bin/sh"`)
     </td>
   </tr>
   <tr>

From bc315326fa70876444cc7bc791a02e619f46304e Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 15 Aug 2022 11:12:41 +0200
Subject: [PATCH 025/112] fix whitespace to please markdown

keep some indentation to ease source readability
---
 doc/manual/src/language/index.md | 886 ++++++++++++++++++-------------
 1 file changed, 531 insertions(+), 355 deletions(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index 724484460..cc71c3143 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -20,370 +20,546 @@ manual](https://nixos.org/nix/manual/#chap-writing-nix-expressions) for
 the rest.
 
 <table>
-  <tr>
-    <th>
-       Example
-    </th>
-    <th>
-      Description
-    </th>
-  </tr>
-  <tr>
-    <td>
-       *Basic values*
-    </td>
-    <td>
+ <tr>
+  <th>
+   Example
+  </th>
+  <th>
+   Description
+  </th>
+ </tr>
+ <tr>
+  <td>
 
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `"hello world"`
-    </td>
-    <td>
-      A string
-    </td>
-    <td>
-       ```
-        ''
-          multi
-           line
-            string
-        ''
-       ```
-    </td>
-    <td>
-      A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`.
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `"hello ${ { a = "world" }.a }"`
 
-       `"1 2 ${3}"`
+   *Basic values*
 
-       `"${pkgs.bash}/bin/sh"`
-    </td>
-    <td>
-      String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/<hash>-bash-<version>/bin/sh"`)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `true`, `false`
-    </td>
-    <td>
-      Booleans
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `null`
-    </td>
-    <td>
-      Null value
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `123`
-    </td>
-    <td>
-      An integer
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `3.141`
-    </td>
-    <td>
-      A floating point number
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `/etc`
-    </td>
-    <td>
-      An absolute path
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `./foo.png`
-    </td>
-    <td>
-      A path relative to the file containing this Nix expression
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `~/.config`
-    </td>
-    <td>
-      A home path. Evaluates to the `"<user's home directory>/.config"`.
-    </td>
-  </tr>
-  <tr>
-    <td>
-       <nixpkgs>
-    </td>
-    <td>
-      Search path. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH).
-    </td>
-  </tr>
-  <tr>
-    <td>
-       *Compound values*
-    </td>
-    <td>
 
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x = 1; y = 2; }`
-    </td>
-    <td>
-      A set with attributes named `x` and `y`
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ foo.bar = 1; }`
-    </td>
-    <td>
-      A nested set, equivalent to `{ foo = { bar = 1; }; }`
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `rec { x = "foo"; y = x + "bar"; }`
-    </td>
-    <td>
-      A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }`
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `[ "foo" "bar" "baz" ]`
+  </td>
+  <td>
 
-       `[ 1 2 3 ]`
 
-       `[ (f 1) { a = 1; b = 2; } [ "c" ] ]`
-    </td>
-    <td>
-      Lists with three elements.
-    </td>
-  </tr>
-  <tr>
-    <td>
-       *Operators*
-    </td>
-    <td>
 
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `"foo" + "bar"`
-    </td>
-    <td>
-      String concatenation
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `1 + 2`
-    </td>
-    <td>
-      Integer addition
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `"foo" == "f" + "oo"`
-    </td>
-    <td>
-      Equality test (evaluates to `true`)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `"foo" != "bar"`
-    </td>
-    <td>
-      Inequality test (evaluates to `true`)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `!true`
-    </td>
-    <td>
-      Boolean negation
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x = 1; y = 2; }.x`
-    </td>
-    <td>
-      Attribute selection (evaluates to `1`)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x = 1; y = 2; }.z or 3`
-    </td>
-    <td>
-      Attribute selection with default (evaluates to `3`)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x = 1; y = 2; } // { z = 3; }`
-    </td>
-    <td>
-      Merge two sets (attributes in the right-hand set taking precedence)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       *Control structures*
-    </td>
-    <td>
+  </td>
+ </tr>
+ <tr>
+  <td>
 
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `if 1 + 1 == 2 then "yes!" else "no!"`
-    </td>
-    <td>
-      Conditional expression
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `assert 1 + 1 == 2; "yes!"`
-    </td>
-    <td>
-      Assertion check (evaluates to `"yes!"`).
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `let x = "foo"; y = "bar"; in x + y`
-    </td>
-    <td>
-      Variable definition
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `with pkgs.lib; head [ 1 2 3 ]`
-    </td>
-    <td>
-      Add all attributes from the given set to the scope (evaluates to `1`)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       *Functions (lambdas)*
-    </td>
-    <td>
+   `"hello world"`
 
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `x: x + 1`
-    </td>
-    <td>
-      A function that expects an integer and returns it increased by 1
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `x: y: x + y`
-    </td>
-    <td>
-      Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum.
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `(x: x + 1) 100`
-    </td>
-    <td>
-      A function call (evaluates to 101)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `let inc = x: x + 1; in inc (inc (inc 100))`
-    </td>
-    <td>
-      A function bound to a variable and subsequently called by name (evaluates to 103)
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x, y }: x + y`
-    </td>
-    <td>
-      A function that expects a set with required attributes `x` and `y` and concatenates them
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x, y ? "bar" }: x + y`
-    </td>
-    <td>
-      A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y`
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x, y, ... }: x + y`
-    </td>
-    <td>
-      A function that expects a set with required attributes `x` and `y` and ignores any other attributes
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `{ x, y } @ args: x + y`
+  </td>
+  <td>
 
-       `args @ { x, y }: x + y`
-    </td>
-    <td>
-      A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args`
-    </td>
-  </tr>
-  <tr>
-    <td>
-       *Built-in functions*
-    </td>
-    <td>
+   A string
 
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `import ./foo.nix`
-    </td>
-    <td>
-      Load and return Nix expression in given file
-    </td>
-  </tr>
-  <tr>
-    <td>
-       `map (x: x + x) [ 1 2 3 ]`
-    </td>
-    <td>
-      Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`)
-    </td>
-  </tr>
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   ```
+   ''
+     multi
+      line
+       string
+   ''
+   ```
+
+  </td>
+  <td>
+
+   A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`.
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `"hello ${ { a = "world" }.a }"`
+
+   `"1 2 ${3}"`
+
+   `"${pkgs.bash}/bin/sh"`
+
+  </td>
+  <td>
+
+   String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/<hash>-bash-<version>/bin/sh"`)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `true`, `false`
+
+  </td>
+  <td>
+
+   Booleans
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `null`
+
+  </td>
+  <td>
+
+   Null value
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `123`
+
+  </td>
+  <td>
+
+   An integer
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `3.141`
+
+  </td>
+  <td>
+
+   A floating point number
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `/etc`
+
+  </td>
+  <td>
+
+   An absolute path
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `./foo.png`
+
+  </td>
+  <td>
+
+   A path relative to the file containing this Nix expression
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `~/.config`
+
+  </td>
+  <td>
+
+   A home path. Evaluates to the `"<user's home directory>/.config"`.
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   <nixpkgs>
+
+  </td>
+  <td>
+
+   Search path. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH).
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   *Compound values*
+
+  </td>
+  <td>
+
+
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x = 1; y = 2; }`
+
+  </td>
+  <td>
+
+   A set with attributes named `x` and `y`
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ foo.bar = 1; }`
+
+  </td>
+  <td>
+
+   A nested set, equivalent to `{ foo = { bar = 1; }; }`
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `rec { x = "foo"; y = x + "bar"; }`
+
+  </td>
+  <td>
+
+   A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }`
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `[ "foo" "bar" "baz" ]`
+
+   `[ 1 2 3 ]`
+
+   `[ (f 1) { a = 1; b = 2; } [ "c" ] ]`
+
+  </td>
+  <td>
+
+   Lists with three elements.
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   *Operators*
+
+  </td>
+  <td>
+
+
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `"foo" + "bar"`
+
+  </td>
+  <td>
+
+   String concatenation
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `1 + 2`
+
+  </td>
+  <td>
+
+   Integer addition
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `"foo" == "f" + "oo"`
+
+  </td>
+  <td>
+
+   Equality test (evaluates to `true`)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `"foo" != "bar"`
+
+  </td>
+  <td>
+
+   Inequality test (evaluates to `true`)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `!true`
+
+  </td>
+  <td>
+
+   Boolean negation
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x = 1; y = 2; }.x`
+
+  </td>
+  <td>
+
+   Attribute selection (evaluates to `1`)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x = 1; y = 2; }.z or 3`
+
+  </td>
+  <td>
+
+   Attribute selection with default (evaluates to `3`)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x = 1; y = 2; } // { z = 3; }`
+
+  </td>
+  <td>
+
+   Merge two sets (attributes in the right-hand set taking precedence)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   *Control structures*
+
+  </td>
+  <td>
+
+
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `if 1 + 1 == 2 then "yes!" else "no!"`
+
+  </td>
+  <td>
+
+   Conditional expression
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `assert 1 + 1 == 2; "yes!"`
+
+  </td>
+  <td>
+
+   Assertion check (evaluates to `"yes!"`).
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `let x = "foo"; y = "bar"; in x + y`
+
+  </td>
+  <td>
+
+   Variable definition
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `with pkgs.lib; head [ 1 2 3 ]`
+
+  </td>
+  <td>
+
+   Add all attributes from the given set to the scope (evaluates to `1`)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   *Functions (lambdas)*
+
+  </td>
+  <td>
+
+
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `x: x + 1`
+
+  </td>
+  <td>
+
+   A function that expects an integer and returns it increased by 1
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `x: y: x + y`
+
+  </td>
+  <td>
+
+   Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum.
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `(x: x + 1) 100`
+
+  </td>
+  <td>
+
+   A function call (evaluates to 101)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `let inc = x: x + 1; in inc (inc (inc 100))`
+
+  </td>
+  <td>
+
+   A function bound to a variable and subsequently called by name (evaluates to 103)
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x, y }: x + y`
+
+  </td>
+  <td>
+
+   A function that expects a set with required attributes `x` and `y` and concatenates them
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x, y ? "bar" }: x + y`
+
+  </td>
+  <td>
+
+   A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y`
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x, y, ... }: x + y`
+
+  </td>
+  <td>
+
+   A function that expects a set with required attributes `x` and `y` and ignores any other attributes
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `{ x, y } @ args: x + y`
+
+   `args @ { x, y }: x + y`
+
+  </td>
+  <td>
+
+   A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args`
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   *Built-in functions*
+
+  </td>
+  <td>
+
+
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `import ./foo.nix`
+
+  </td>
+  <td>
+
+   Load and return Nix expression in given file
+
+  </td>
+ </tr>
+ <tr>
+  <td>
+
+   `map (x: x + x) [ 1 2 3 ]`
+
+  </td>
+  <td>
+
+   Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`)
+
+  </td>
+ </tr>
 </table>

From 71e9c2869429bb711bf728f4a7acd14f5c2e1eeb Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 15 Aug 2022 11:15:03 +0200
Subject: [PATCH 026/112] reword introduction to overview

---
 doc/manual/src/language/index.md | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index cc71c3143..cea17bedc 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -11,13 +11,9 @@ packages, compositions of packages, and the variability within packages.
 
 This section presents the various features of the language.
 
-# Syntax Summary
+# Overview
 
-Below is a summary of the most important syntactic constructs in the Nix
-expression language. It's not complete. In particular, there are many
-other built-in functions. See the [Nix
-manual](https://nixos.org/nix/manual/#chap-writing-nix-expressions) for
-the rest.
+This is an incomplete overview of language features, by example.
 
 <table>
  <tr>

From f35b3aa47bc6edc2a929e1803c8429113c34012b Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 24 Aug 2022 08:48:34 +0200
Subject: [PATCH 027/112] do not use unwarranted pkgs in example

---
 doc/manual/src/language/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index cea17bedc..83484dced 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -403,7 +403,7 @@ This is an incomplete overview of language features, by example.
  <tr>
   <td>
 
-   `with pkgs.lib; head [ 1 2 3 ]`
+   `with builtins; head [ 1 2 3 ]`
 
   </td>
   <td>

From 59be1e500a24c9e7970fabcc97bad5280300061c Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 5 Sep 2022 22:37:22 +0200
Subject: [PATCH 028/112] generalize anchor redirects

renaming section headers and changing manually set `id`s will break URLs
in the wild.

this change allows keeping track of all changes to ensure backwards
compatibility.
---
 doc/manual/redirects.js | 735 ++++++++++++++++++++++------------------
 1 file changed, 409 insertions(+), 326 deletions(-)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index 167e221b8..45fbfffab 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -1,330 +1,413 @@
-// Redirects from old DocBook manual.
+// Redirect rules for anchors ensure backwards compatibility of URLs.
+// This must be done on the client side, as web servers do not see the anchor part of the URL.
+
+// Redirections are declared as follows:
+// Each entry has as key the matched URL path relative to the mdBook document root.
+// Each entry is a set of key-value pairs, where
+// - keys are anchors on to the matched path.
+// - values are redirection targets relative to the current path.
+
 var redirects = {
-  "#part-advanced-topics": "advanced-topics/advanced-topics.html",
-  "#chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html",
-  "#chap-diff-hook": "advanced-topics/diff-hook.html",
-  "#check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered",
-  "#chap-distributed-builds": "advanced-topics/distributed-builds.html",
-  "#chap-post-build-hook": "advanced-topics/post-build-hook.html",
-  "#chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
-  "#part-command-ref": "command-ref/command-ref.html",
-  "#conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
-  "#conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",
-  "#conf-allowed-uris": "command-ref/conf-file.html#conf-allowed-uris",
-  "#conf-allowed-users": "command-ref/conf-file.html#conf-allowed-users",
-  "#conf-auto-optimise-store": "command-ref/conf-file.html#conf-auto-optimise-store",
-  "#conf-binary-cache-public-keys": "command-ref/conf-file.html#conf-binary-cache-public-keys",
-  "#conf-binary-caches": "command-ref/conf-file.html#conf-binary-caches",
-  "#conf-build-compress-log": "command-ref/conf-file.html#conf-build-compress-log",
-  "#conf-build-cores": "command-ref/conf-file.html#conf-build-cores",
-  "#conf-build-extra-chroot-dirs": "command-ref/conf-file.html#conf-build-extra-chroot-dirs",
-  "#conf-build-extra-sandbox-paths": "command-ref/conf-file.html#conf-build-extra-sandbox-paths",
-  "#conf-build-fallback": "command-ref/conf-file.html#conf-build-fallback",
-  "#conf-build-max-jobs": "command-ref/conf-file.html#conf-build-max-jobs",
-  "#conf-build-max-log-size": "command-ref/conf-file.html#conf-build-max-log-size",
-  "#conf-build-max-silent-time": "command-ref/conf-file.html#conf-build-max-silent-time",
-  "#conf-build-repeat": "command-ref/conf-file.html#conf-build-repeat",
-  "#conf-build-timeout": "command-ref/conf-file.html#conf-build-timeout",
-  "#conf-build-use-chroot": "command-ref/conf-file.html#conf-build-use-chroot",
-  "#conf-build-use-sandbox": "command-ref/conf-file.html#conf-build-use-sandbox",
-  "#conf-build-use-substitutes": "command-ref/conf-file.html#conf-build-use-substitutes",
-  "#conf-build-users-group": "command-ref/conf-file.html#conf-build-users-group",
-  "#conf-builders": "command-ref/conf-file.html#conf-builders",
-  "#conf-builders-use-substitutes": "command-ref/conf-file.html#conf-builders-use-substitutes",
-  "#conf-compress-build-log": "command-ref/conf-file.html#conf-compress-build-log",
-  "#conf-connect-timeout": "command-ref/conf-file.html#conf-connect-timeout",
-  "#conf-cores": "command-ref/conf-file.html#conf-cores",
-  "#conf-diff-hook": "command-ref/conf-file.html#conf-diff-hook",
-  "#conf-enforce-determinism": "command-ref/conf-file.html#conf-enforce-determinism",
-  "#conf-env-keep-derivations": "command-ref/conf-file.html#conf-env-keep-derivations",
-  "#conf-extra-binary-caches": "command-ref/conf-file.html#conf-extra-binary-caches",
-  "#conf-extra-platforms": "command-ref/conf-file.html#conf-extra-platforms",
-  "#conf-extra-sandbox-paths": "command-ref/conf-file.html#conf-extra-sandbox-paths",
-  "#conf-extra-substituters": "command-ref/conf-file.html#conf-extra-substituters",
-  "#conf-fallback": "command-ref/conf-file.html#conf-fallback",
-  "#conf-fsync-metadata": "command-ref/conf-file.html#conf-fsync-metadata",
-  "#conf-gc-keep-derivations": "command-ref/conf-file.html#conf-gc-keep-derivations",
-  "#conf-gc-keep-outputs": "command-ref/conf-file.html#conf-gc-keep-outputs",
-  "#conf-hashed-mirrors": "command-ref/conf-file.html#conf-hashed-mirrors",
-  "#conf-http-connections": "command-ref/conf-file.html#conf-http-connections",
-  "#conf-keep-build-log": "command-ref/conf-file.html#conf-keep-build-log",
-  "#conf-keep-derivations": "command-ref/conf-file.html#conf-keep-derivations",
-  "#conf-keep-env-derivations": "command-ref/conf-file.html#conf-keep-env-derivations",
-  "#conf-keep-outputs": "command-ref/conf-file.html#conf-keep-outputs",
-  "#conf-max-build-log-size": "command-ref/conf-file.html#conf-max-build-log-size",
-  "#conf-max-free": "command-ref/conf-file.html#conf-max-free",
-  "#conf-max-jobs": "command-ref/conf-file.html#conf-max-jobs",
-  "#conf-max-silent-time": "command-ref/conf-file.html#conf-max-silent-time",
-  "#conf-min-free": "command-ref/conf-file.html#conf-min-free",
-  "#conf-narinfo-cache-negative-ttl": "command-ref/conf-file.html#conf-narinfo-cache-negative-ttl",
-  "#conf-narinfo-cache-positive-ttl": "command-ref/conf-file.html#conf-narinfo-cache-positive-ttl",
-  "#conf-netrc-file": "command-ref/conf-file.html#conf-netrc-file",
-  "#conf-plugin-files": "command-ref/conf-file.html#conf-plugin-files",
-  "#conf-post-build-hook": "command-ref/conf-file.html#conf-post-build-hook",
-  "#conf-pre-build-hook": "command-ref/conf-file.html#conf-pre-build-hook",
-  "#conf-repeat": "command-ref/conf-file.html#conf-repeat",
-  "#conf-require-sigs": "command-ref/conf-file.html#conf-require-sigs",
-  "#conf-restrict-eval": "command-ref/conf-file.html#conf-restrict-eval",
-  "#conf-run-diff-hook": "command-ref/conf-file.html#conf-run-diff-hook",
-  "#conf-sandbox": "command-ref/conf-file.html#conf-sandbox",
-  "#conf-sandbox-dev-shm-size": "command-ref/conf-file.html#conf-sandbox-dev-shm-size",
-  "#conf-sandbox-paths": "command-ref/conf-file.html#conf-sandbox-paths",
-  "#conf-secret-key-files": "command-ref/conf-file.html#conf-secret-key-files",
-  "#conf-show-trace": "command-ref/conf-file.html#conf-show-trace",
-  "#conf-stalled-download-timeout": "command-ref/conf-file.html#conf-stalled-download-timeout",
-  "#conf-substitute": "command-ref/conf-file.html#conf-substitute",
-  "#conf-substituters": "command-ref/conf-file.html#conf-substituters",
-  "#conf-system": "command-ref/conf-file.html#conf-system",
-  "#conf-system-features": "command-ref/conf-file.html#conf-system-features",
-  "#conf-tarball-ttl": "command-ref/conf-file.html#conf-tarball-ttl",
-  "#conf-timeout": "command-ref/conf-file.html#conf-timeout",
-  "#conf-trace-function-calls": "command-ref/conf-file.html#conf-trace-function-calls",
-  "#conf-trusted-binary-caches": "command-ref/conf-file.html#conf-trusted-binary-caches",
-  "#conf-trusted-public-keys": "command-ref/conf-file.html#conf-trusted-public-keys",
-  "#conf-trusted-substituters": "command-ref/conf-file.html#conf-trusted-substituters",
-  "#conf-trusted-users": "command-ref/conf-file.html#conf-trusted-users",
-  "#extra-sandbox-paths": "command-ref/conf-file.html#extra-sandbox-paths",
-  "#sec-conf-file": "command-ref/conf-file.html",
-  "#env-NIX_PATH": "command-ref/env-common.html#env-NIX_PATH",
-  "#env-common": "command-ref/env-common.html",
-  "#envar-remote": "command-ref/env-common.html#env-NIX_REMOTE",
-  "#sec-common-env": "command-ref/env-common.html",
-  "#ch-files": "command-ref/files.html",
-  "#ch-main-commands": "command-ref/main-commands.html",
-  "#opt-out-link": "command-ref/nix-build.html#opt-out-link",
-  "#sec-nix-build": "command-ref/nix-build.html",
-  "#sec-nix-channel": "command-ref/nix-channel.html",
-  "#sec-nix-collect-garbage": "command-ref/nix-collect-garbage.html",
-  "#sec-nix-copy-closure": "command-ref/nix-copy-closure.html",
-  "#sec-nix-daemon": "command-ref/nix-daemon.html",
-  "#refsec-nix-env-install-examples": "command-ref/nix-env.html#examples",
-  "#rsec-nix-env-install": "command-ref/nix-env.html#operation---install",
-  "#rsec-nix-env-set": "command-ref/nix-env.html#operation---set",
-  "#rsec-nix-env-set-flag": "command-ref/nix-env.html#operation---set-flag",
-  "#rsec-nix-env-upgrade": "command-ref/nix-env.html#operation---upgrade",
-  "#sec-nix-env": "command-ref/nix-env.html",
-  "#ssec-version-comparisons": "command-ref/nix-env.html#versions",
-  "#sec-nix-hash": "command-ref/nix-hash.html",
-  "#sec-nix-instantiate": "command-ref/nix-instantiate.html",
-  "#sec-nix-prefetch-url": "command-ref/nix-prefetch-url.html",
-  "#sec-nix-shell": "command-ref/nix-shell.html",
-  "#ssec-nix-shell-shebang": "command-ref/nix-shell.html#use-as-a--interpreter",
-  "#nixref-queries": "command-ref/nix-store.html#queries",
-  "#opt-add-root": "command-ref/nix-store.html#opt-add-root",
-  "#refsec-nix-store-dump": "command-ref/nix-store.html#operation---dump",
-  "#refsec-nix-store-export": "command-ref/nix-store.html#operation---export",
-  "#refsec-nix-store-import": "command-ref/nix-store.html#operation---import",
-  "#refsec-nix-store-query": "command-ref/nix-store.html#operation---query",
-  "#refsec-nix-store-verify": "command-ref/nix-store.html#operation---verify",
-  "#rsec-nix-store-gc": "command-ref/nix-store.html#operation---gc",
-  "#rsec-nix-store-generate-binary-cache-key": "command-ref/nix-store.html#operation---generate-binary-cache-key",
-  "#rsec-nix-store-realise": "command-ref/nix-store.html#operation---realise",
-  "#rsec-nix-store-serve": "command-ref/nix-store.html#operation---serve",
-  "#sec-nix-store": "command-ref/nix-store.html",
-  "#opt-I": "command-ref/opt-common.html#opt-I",
-  "#opt-attr": "command-ref/opt-common.html#opt-attr",
-  "#opt-common": "command-ref/opt-common.html",
-  "#opt-cores": "command-ref/opt-common.html#opt-cores",
-  "#opt-log-format": "command-ref/opt-common.html#opt-log-format",
-  "#opt-max-jobs": "command-ref/opt-common.html#opt-max-jobs",
-  "#opt-max-silent-time": "command-ref/opt-common.html#opt-max-silent-time",
-  "#opt-timeout": "command-ref/opt-common.html#opt-timeout",
-  "#sec-common-options": "command-ref/opt-common.html",
-  "#ch-utilities": "command-ref/utilities.html",
-  "#chap-hacking": "contributing/hacking.html",
-  "#adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes",
-  "#adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences",
-  "#adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites",
-  "#adv-attr-disallowedReferences": "language/advanced-attributes.html#adv-attr-disallowedReferences",
-  "#adv-attr-disallowedRequisites": "language/advanced-attributes.html#adv-attr-disallowedRequisites",
-  "#adv-attr-exportReferencesGraph": "language/advanced-attributes.html#adv-attr-exportReferencesGraph",
-  "#adv-attr-impureEnvVars": "language/advanced-attributes.html#adv-attr-impureEnvVars",
-  "#adv-attr-outputHash": "language/advanced-attributes.html#adv-attr-outputHash",
-  "#adv-attr-outputHashAlgo": "language/advanced-attributes.html#adv-attr-outputHashAlgo",
-  "#adv-attr-outputHashMode": "language/advanced-attributes.html#adv-attr-outputHashMode",
-  "#adv-attr-passAsFile": "language/advanced-attributes.html#adv-attr-passAsFile",
-  "#adv-attr-preferLocalBuild": "language/advanced-attributes.html#adv-attr-preferLocalBuild",
-  "#fixed-output-drvs": "language/advanced-attributes.html#adv-attr-outputHash",
-  "#sec-advanced-attributes": "language/advanced-attributes.html",
-  "#builtin-abort": "language/builtins.html#builtins-abort",
-  "#builtin-add": "language/builtins.html#builtins-add",
-  "#builtin-all": "language/builtins.html#builtins-all",
-  "#builtin-any": "language/builtins.html#builtins-any",
-  "#builtin-attrNames": "language/builtins.html#builtins-attrNames",
-  "#builtin-attrValues": "language/builtins.html#builtins-attrValues",
-  "#builtin-baseNameOf": "language/builtins.html#builtins-baseNameOf",
-  "#builtin-bitAnd": "language/builtins.html#builtins-bitAnd",
-  "#builtin-bitOr": "language/builtins.html#builtins-bitOr",
-  "#builtin-bitXor": "language/builtins.html#builtins-bitXor",
-  "#builtin-builtins": "language/builtins.html#builtins-builtins",
-  "#builtin-compareVersions": "language/builtins.html#builtins-compareVersions",
-  "#builtin-concatLists": "language/builtins.html#builtins-concatLists",
-  "#builtin-concatStringsSep": "language/builtins.html#builtins-concatStringsSep",
-  "#builtin-currentSystem": "language/builtins.html#builtins-currentSystem",
-  "#builtin-deepSeq": "language/builtins.html#builtins-deepSeq",
-  "#builtin-derivation": "language/builtins.html#builtins-derivation",
-  "#builtin-dirOf": "language/builtins.html#builtins-dirOf",
-  "#builtin-div": "language/builtins.html#builtins-div",
-  "#builtin-elem": "language/builtins.html#builtins-elem",
-  "#builtin-elemAt": "language/builtins.html#builtins-elemAt",
-  "#builtin-fetchGit": "language/builtins.html#builtins-fetchGit",
-  "#builtin-fetchTarball": "language/builtins.html#builtins-fetchTarball",
-  "#builtin-fetchurl": "language/builtins.html#builtins-fetchurl",
-  "#builtin-filterSource": "language/builtins.html#builtins-filterSource",
-  "#builtin-foldl-prime": "language/builtins.html#builtins-foldl-prime",
-  "#builtin-fromJSON": "language/builtins.html#builtins-fromJSON",
-  "#builtin-functionArgs": "language/builtins.html#builtins-functionArgs",
-  "#builtin-genList": "language/builtins.html#builtins-genList",
-  "#builtin-getAttr": "language/builtins.html#builtins-getAttr",
-  "#builtin-getEnv": "language/builtins.html#builtins-getEnv",
-  "#builtin-hasAttr": "language/builtins.html#builtins-hasAttr",
-  "#builtin-hashFile": "language/builtins.html#builtins-hashFile",
-  "#builtin-hashString": "language/builtins.html#builtins-hashString",
-  "#builtin-head": "language/builtins.html#builtins-head",
-  "#builtin-import": "language/builtins.html#builtins-import",
-  "#builtin-intersectAttrs": "language/builtins.html#builtins-intersectAttrs",
-  "#builtin-isAttrs": "language/builtins.html#builtins-isAttrs",
-  "#builtin-isBool": "language/builtins.html#builtins-isBool",
-  "#builtin-isFloat": "language/builtins.html#builtins-isFloat",
-  "#builtin-isFunction": "language/builtins.html#builtins-isFunction",
-  "#builtin-isInt": "language/builtins.html#builtins-isInt",
-  "#builtin-isList": "language/builtins.html#builtins-isList",
-  "#builtin-isNull": "language/builtins.html#builtins-isNull",
-  "#builtin-isString": "language/builtins.html#builtins-isString",
-  "#builtin-length": "language/builtins.html#builtins-length",
-  "#builtin-lessThan": "language/builtins.html#builtins-lessThan",
-  "#builtin-listToAttrs": "language/builtins.html#builtins-listToAttrs",
-  "#builtin-map": "language/builtins.html#builtins-map",
-  "#builtin-match": "language/builtins.html#builtins-match",
-  "#builtin-mul": "language/builtins.html#builtins-mul",
-  "#builtin-parseDrvName": "language/builtins.html#builtins-parseDrvName",
-  "#builtin-path": "language/builtins.html#builtins-path",
-  "#builtin-pathExists": "language/builtins.html#builtins-pathExists",
-  "#builtin-placeholder": "language/builtins.html#builtins-placeholder",
-  "#builtin-readDir": "language/builtins.html#builtins-readDir",
-  "#builtin-readFile": "language/builtins.html#builtins-readFile",
-  "#builtin-removeAttrs": "language/builtins.html#builtins-removeAttrs",
-  "#builtin-replaceStrings": "language/builtins.html#builtins-replaceStrings",
-  "#builtin-seq": "language/builtins.html#builtins-seq",
-  "#builtin-sort": "language/builtins.html#builtins-sort",
-  "#builtin-split": "language/builtins.html#builtins-split",
-  "#builtin-splitVersion": "language/builtins.html#builtins-splitVersion",
-  "#builtin-stringLength": "language/builtins.html#builtins-stringLength",
-  "#builtin-sub": "language/builtins.html#builtins-sub",
-  "#builtin-substring": "language/builtins.html#builtins-substring",
-  "#builtin-tail": "language/builtins.html#builtins-tail",
-  "#builtin-throw": "language/builtins.html#builtins-throw",
-  "#builtin-toFile": "language/builtins.html#builtins-toFile",
-  "#builtin-toJSON": "language/builtins.html#builtins-toJSON",
-  "#builtin-toPath": "language/builtins.html#builtins-toPath",
-  "#builtin-toString": "language/builtins.html#builtins-toString",
-  "#builtin-toXML": "language/builtins.html#builtins-toXML",
-  "#builtin-trace": "language/builtins.html#builtins-trace",
-  "#builtin-tryEval": "language/builtins.html#builtins-tryEval",
-  "#builtin-typeOf": "language/builtins.html#builtins-typeOf",
-  "#ssec-builtins": "language/builtins.html",
-  "#attr-system": "language/derivations.html#attr-system",
-  "#ssec-derivation": "language/derivations.html",
-  "#ch-expression-language": "language/index.html",
-  "#sec-constructs": "language/constructs.html",
-  "#sect-let-language": "language/constructs.html#let-language",
-  "#ss-functions": "language/constructs.html#functions",
-  "#sec-language-operators": "language/operators.html",
-  "#table-operators": "language/operators.html",
-  "#ssec-values": "language/values.html",
-  "#gloss-closure": "glossary.html#gloss-closure",
-  "#gloss-derivation": "glossary.html#gloss-derivation",
-  "#gloss-deriver": "glossary.html#gloss-deriver",
-  "#gloss-nar": "glossary.html#gloss-nar",
-  "#gloss-output-path": "glossary.html#gloss-output-path",
-  "#gloss-profile": "glossary.html#gloss-profile",
-  "#gloss-reachable": "glossary.html#gloss-reachable",
-  "#gloss-reference": "glossary.html#gloss-reference",
-  "#gloss-substitute": "glossary.html#gloss-substitute",
-  "#gloss-user-env": "glossary.html#gloss-user-env",
-  "#gloss-validity": "glossary.html#gloss-validity",
-  "#part-glossary": "glossary.html",
-  "#sec-building-source": "installation/building-source.html",
-  "#ch-env-variables": "installation/env-variables.html",
-  "#sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables",
-  "#sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file",
-  "#sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file-with-macos-and-the-nix-daemon",
-  "#chap-installation": "installation/installation.html",
-  "#ch-installing-binary": "installation/installing-binary.html",
-  "#sect-macos-installation": "installation/installing-binary.html#macos-installation",
-  "#sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation",
-  "#sect-macos-installation-encrypted-volume": "installation/installing-binary.html#macos-installation",
-  "#sect-macos-installation-recommended-notes": "installation/installing-binary.html#macos-installation",
-  "#sect-macos-installation-symlink": "installation/installing-binary.html#macos-installation",
-  "#sect-multi-user-installation": "installation/installing-binary.html#multi-user-installation",
-  "#sect-nix-install-binary-tarball": "installation/installing-binary.html#installing-from-a-binary-tarball",
-  "#sect-nix-install-pinned-version-url": "installation/installing-binary.html#installing-a-pinned-nix-version-from-a-url",
-  "#sect-single-user-installation": "installation/installing-binary.html#single-user-installation",
-  "#ch-installing-source": "installation/installing-source.html",
-  "#ssec-multi-user": "installation/multi-user.html",
-  "#ch-nix-security": "installation/nix-security.html",
-  "#sec-obtaining-source": "installation/obtaining-source.html",
-  "#sec-prerequisites-source": "installation/prerequisites-source.html",
-  "#sec-single-user": "installation/single-user.html",
-  "#ch-supported-platforms": "installation/supported-platforms.html",
-  "#ch-upgrading-nix": "installation/upgrading.html",
-  "#ch-about-nix": "introduction.html",
-  "#chap-introduction": "introduction.html",
-  "#ch-basic-package-mgmt": "package-management/basic-package-mgmt.html",
-  "#ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html",
-  "#sec-channels": "package-management/channels.html",
-  "#ssec-copy-closure": "package-management/copy-closure.html",
-  "#sec-garbage-collection": "package-management/garbage-collection.html",
-  "#ssec-gc-roots": "package-management/garbage-collector-roots.html",
-  "#chap-package-management": "package-management/package-management.html",
-  "#sec-profiles": "package-management/profiles.html",
-  "#ssec-s3-substituter": "package-management/s3-substituter.html",
-  "#ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
-  "#ssec-s3-substituter-authenticated-reads": "package-management/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
-  "#ssec-s3-substituter-authenticated-writes": "package-management/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
-  "#sec-sharing-packages": "package-management/sharing-packages.html",
-  "#ssec-ssh-substituter": "package-management/ssh-substituter.html",
-  "#chap-quick-start": "quick-start.html",
-  "#sec-relnotes": "release-notes/release-notes.html",
-  "#ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html",
-  "#ch-relnotes-0.10": "release-notes/rl-0.10.html",
-  "#ssec-relnotes-0.11": "release-notes/rl-0.11.html",
-  "#ssec-relnotes-0.12": "release-notes/rl-0.12.html",
-  "#ssec-relnotes-0.13": "release-notes/rl-0.13.html",
-  "#ssec-relnotes-0.14": "release-notes/rl-0.14.html",
-  "#ssec-relnotes-0.15": "release-notes/rl-0.15.html",
-  "#ssec-relnotes-0.16": "release-notes/rl-0.16.html",
-  "#ch-relnotes-0.5": "release-notes/rl-0.5.html",
-  "#ch-relnotes-0.6": "release-notes/rl-0.6.html",
-  "#ch-relnotes-0.7": "release-notes/rl-0.7.html",
-  "#ch-relnotes-0.8.1": "release-notes/rl-0.8.1.html",
-  "#ch-relnotes-0.8": "release-notes/rl-0.8.html",
-  "#ch-relnotes-0.9.1": "release-notes/rl-0.9.1.html",
-  "#ch-relnotes-0.9.2": "release-notes/rl-0.9.2.html",
-  "#ch-relnotes-0.9": "release-notes/rl-0.9.html",
-  "#ssec-relnotes-1.0": "release-notes/rl-1.0.html",
-  "#ssec-relnotes-1.1": "release-notes/rl-1.1.html",
-  "#ssec-relnotes-1.10": "release-notes/rl-1.10.html",
-  "#ssec-relnotes-1.11.10": "release-notes/rl-1.11.10.html",
-  "#ssec-relnotes-1.11": "release-notes/rl-1.11.html",
-  "#ssec-relnotes-1.2": "release-notes/rl-1.2.html",
-  "#ssec-relnotes-1.3": "release-notes/rl-1.3.html",
-  "#ssec-relnotes-1.4": "release-notes/rl-1.4.html",
-  "#ssec-relnotes-1.5.1": "release-notes/rl-1.5.1.html",
-  "#ssec-relnotes-1.5.2": "release-notes/rl-1.5.2.html",
-  "#ssec-relnotes-1.5": "release-notes/rl-1.5.html",
-  "#ssec-relnotes-1.6.1": "release-notes/rl-1.6.1.html",
-  "#ssec-relnotes-1.6.0": "release-notes/rl-1.6.html",
-  "#ssec-relnotes-1.7": "release-notes/rl-1.7.html",
-  "#ssec-relnotes-1.8": "release-notes/rl-1.8.html",
-  "#ssec-relnotes-1.9": "release-notes/rl-1.9.html",
-  "#ssec-relnotes-2.0": "release-notes/rl-2.0.html",
-  "#ssec-relnotes-2.1": "release-notes/rl-2.1.html",
-  "#ssec-relnotes-2.2": "release-notes/rl-2.2.html",
-  "#ssec-relnotes-2.3": "release-notes/rl-2.3.html"
+ "index.html": {
+    "part-advanced-topics": "advanced-topics/advanced-topics.html",
+    "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html",
+    "chap-diff-hook": "advanced-topics/diff-hook.html",
+    "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered",
+    "chap-distributed-builds": "advanced-topics/distributed-builds.html",
+    "chap-post-build-hook": "advanced-topics/post-build-hook.html",
+    "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
+    "part-command-ref": "command-ref/command-ref.html",
+    "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
+    "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",
+    "conf-allowed-uris": "command-ref/conf-file.html#conf-allowed-uris",
+    "conf-allowed-users": "command-ref/conf-file.html#conf-allowed-users",
+    "conf-auto-optimise-store": "command-ref/conf-file.html#conf-auto-optimise-store",
+    "conf-binary-cache-public-keys": "command-ref/conf-file.html#conf-binary-cache-public-keys",
+    "conf-binary-caches": "command-ref/conf-file.html#conf-binary-caches",
+    "conf-build-compress-log": "command-ref/conf-file.html#conf-build-compress-log",
+    "conf-build-cores": "command-ref/conf-file.html#conf-build-cores",
+    "conf-build-extra-chroot-dirs": "command-ref/conf-file.html#conf-build-extra-chroot-dirs",
+    "conf-build-extra-sandbox-paths": "command-ref/conf-file.html#conf-build-extra-sandbox-paths",
+    "conf-build-fallback": "command-ref/conf-file.html#conf-build-fallback",
+    "conf-build-max-jobs": "command-ref/conf-file.html#conf-build-max-jobs",
+    "conf-build-max-log-size": "command-ref/conf-file.html#conf-build-max-log-size",
+    "conf-build-max-silent-time": "command-ref/conf-file.html#conf-build-max-silent-time",
+    "conf-build-repeat": "command-ref/conf-file.html#conf-build-repeat",
+    "conf-build-timeout": "command-ref/conf-file.html#conf-build-timeout",
+    "conf-build-use-chroot": "command-ref/conf-file.html#conf-build-use-chroot",
+    "conf-build-use-sandbox": "command-ref/conf-file.html#conf-build-use-sandbox",
+    "conf-build-use-substitutes": "command-ref/conf-file.html#conf-build-use-substitutes",
+    "conf-build-users-group": "command-ref/conf-file.html#conf-build-users-group",
+    "conf-builders": "command-ref/conf-file.html#conf-builders",
+    "conf-builders-use-substitutes": "command-ref/conf-file.html#conf-builders-use-substitutes",
+    "conf-compress-build-log": "command-ref/conf-file.html#conf-compress-build-log",
+    "conf-connect-timeout": "command-ref/conf-file.html#conf-connect-timeout",
+    "conf-cores": "command-ref/conf-file.html#conf-cores",
+    "conf-diff-hook": "command-ref/conf-file.html#conf-diff-hook",
+    "conf-enforce-determinism": "command-ref/conf-file.html#conf-enforce-determinism",
+    "conf-env-keep-derivations": "command-ref/conf-file.html#conf-env-keep-derivations",
+    "conf-extra-binary-caches": "command-ref/conf-file.html#conf-extra-binary-caches",
+    "conf-extra-platforms": "command-ref/conf-file.html#conf-extra-platforms",
+    "conf-extra-sandbox-paths": "command-ref/conf-file.html#conf-extra-sandbox-paths",
+    "conf-extra-substituters": "command-ref/conf-file.html#conf-extra-substituters",
+    "conf-fallback": "command-ref/conf-file.html#conf-fallback",
+    "conf-fsync-metadata": "command-ref/conf-file.html#conf-fsync-metadata",
+    "conf-gc-keep-derivations": "command-ref/conf-file.html#conf-gc-keep-derivations",
+    "conf-gc-keep-outputs": "command-ref/conf-file.html#conf-gc-keep-outputs",
+    "conf-hashed-mirrors": "command-ref/conf-file.html#conf-hashed-mirrors",
+    "conf-http-connections": "command-ref/conf-file.html#conf-http-connections",
+    "conf-keep-build-log": "command-ref/conf-file.html#conf-keep-build-log",
+    "conf-keep-derivations": "command-ref/conf-file.html#conf-keep-derivations",
+    "conf-keep-env-derivations": "command-ref/conf-file.html#conf-keep-env-derivations",
+    "conf-keep-outputs": "command-ref/conf-file.html#conf-keep-outputs",
+    "conf-max-build-log-size": "command-ref/conf-file.html#conf-max-build-log-size",
+    "conf-max-free": "command-ref/conf-file.html#conf-max-free",
+    "conf-max-jobs": "command-ref/conf-file.html#conf-max-jobs",
+    "conf-max-silent-time": "command-ref/conf-file.html#conf-max-silent-time",
+    "conf-min-free": "command-ref/conf-file.html#conf-min-free",
+    "conf-narinfo-cache-negative-ttl": "command-ref/conf-file.html#conf-narinfo-cache-negative-ttl",
+    "conf-narinfo-cache-positive-ttl": "command-ref/conf-file.html#conf-narinfo-cache-positive-ttl",
+    "conf-netrc-file": "command-ref/conf-file.html#conf-netrc-file",
+    "conf-plugin-files": "command-ref/conf-file.html#conf-plugin-files",
+    "conf-post-build-hook": "command-ref/conf-file.html#conf-post-build-hook",
+    "conf-pre-build-hook": "command-ref/conf-file.html#conf-pre-build-hook",
+    "conf-repeat": "command-ref/conf-file.html#conf-repeat",
+    "conf-require-sigs": "command-ref/conf-file.html#conf-require-sigs",
+    "conf-restrict-eval": "command-ref/conf-file.html#conf-restrict-eval",
+    "conf-run-diff-hook": "command-ref/conf-file.html#conf-run-diff-hook",
+    "conf-sandbox": "command-ref/conf-file.html#conf-sandbox",
+    "conf-sandbox-dev-shm-size": "command-ref/conf-file.html#conf-sandbox-dev-shm-size",
+    "conf-sandbox-paths": "command-ref/conf-file.html#conf-sandbox-paths",
+    "conf-secret-key-files": "command-ref/conf-file.html#conf-secret-key-files",
+    "conf-show-trace": "command-ref/conf-file.html#conf-show-trace",
+    "conf-stalled-download-timeout": "command-ref/conf-file.html#conf-stalled-download-timeout",
+    "conf-substitute": "command-ref/conf-file.html#conf-substitute",
+    "conf-substituters": "command-ref/conf-file.html#conf-substituters",
+    "conf-system": "command-ref/conf-file.html#conf-system",
+    "conf-system-features": "command-ref/conf-file.html#conf-system-features",
+    "conf-tarball-ttl": "command-ref/conf-file.html#conf-tarball-ttl",
+    "conf-timeout": "command-ref/conf-file.html#conf-timeout",
+    "conf-trace-function-calls": "command-ref/conf-file.html#conf-trace-function-calls",
+    "conf-trusted-binary-caches": "command-ref/conf-file.html#conf-trusted-binary-caches",
+    "conf-trusted-public-keys": "command-ref/conf-file.html#conf-trusted-public-keys",
+    "conf-trusted-substituters": "command-ref/conf-file.html#conf-trusted-substituters",
+    "conf-trusted-users": "command-ref/conf-file.html#conf-trusted-users",
+    "extra-sandbox-paths": "command-ref/conf-file.html#extra-sandbox-paths",
+    "sec-conf-file": "command-ref/conf-file.html",
+    "env-NIX_PATH": "command-ref/env-common.html#env-NIX_PATH",
+    "env-common": "command-ref/env-common.html",
+    "envar-remote": "command-ref/env-common.html#env-NIX_REMOTE",
+    "sec-common-env": "command-ref/env-common.html",
+    "ch-files": "command-ref/files.html",
+    "ch-main-commands": "command-ref/main-commands.html",
+    "opt-out-link": "command-ref/nix-build.html#opt-out-link",
+    "sec-nix-build": "command-ref/nix-build.html",
+    "sec-nix-channel": "command-ref/nix-channel.html",
+    "sec-nix-collect-garbage": "command-ref/nix-collect-garbage.html",
+    "sec-nix-copy-closure": "command-ref/nix-copy-closure.html",
+    "sec-nix-daemon": "command-ref/nix-daemon.html",
+    "refsec-nix-env-install-examples": "command-ref/nix-env.html#examples",
+    "rsec-nix-env-install": "command-ref/nix-env.html#operation---install",
+    "rsec-nix-env-set": "command-ref/nix-env.html#operation---set",
+    "rsec-nix-env-set-flag": "command-ref/nix-env.html#operation---set-flag",
+    "rsec-nix-env-upgrade": "command-ref/nix-env.html#operation---upgrade",
+    "sec-nix-env": "command-ref/nix-env.html",
+    "ssec-version-comparisons": "command-ref/nix-env.html#versions",
+    "sec-nix-hash": "command-ref/nix-hash.html",
+    "sec-nix-instantiate": "command-ref/nix-instantiate.html",
+    "sec-nix-prefetch-url": "command-ref/nix-prefetch-url.html",
+    "sec-nix-shell": "command-ref/nix-shell.html",
+    "ssec-nix-shell-shebang": "command-ref/nix-shell.html#use-as-a--interpreter",
+    "nixref-queries": "command-ref/nix-store.html#queries",
+    "opt-add-root": "command-ref/nix-store.html#opt-add-root",
+    "refsec-nix-store-dump": "command-ref/nix-store.html#operation---dump",
+    "refsec-nix-store-export": "command-ref/nix-store.html#operation---export",
+    "refsec-nix-store-import": "command-ref/nix-store.html#operation---import",
+    "refsec-nix-store-query": "command-ref/nix-store.html#operation---query",
+    "refsec-nix-store-verify": "command-ref/nix-store.html#operation---verify",
+    "rsec-nix-store-gc": "command-ref/nix-store.html#operation---gc",
+    "rsec-nix-store-generate-binary-cache-key": "command-ref/nix-store.html#operation---generate-binary-cache-key",
+    "rsec-nix-store-realise": "command-ref/nix-store.html#operation---realise",
+    "rsec-nix-store-serve": "command-ref/nix-store.html#operation---serve",
+    "sec-nix-store": "command-ref/nix-store.html",
+    "opt-I": "command-ref/opt-common.html#opt-I",
+    "opt-attr": "command-ref/opt-common.html#opt-attr",
+    "opt-common": "command-ref/opt-common.html",
+    "opt-cores": "command-ref/opt-common.html#opt-cores",
+    "opt-log-format": "command-ref/opt-common.html#opt-log-format",
+    "opt-max-jobs": "command-ref/opt-common.html#opt-max-jobs",
+    "opt-max-silent-time": "command-ref/opt-common.html#opt-max-silent-time",
+    "opt-timeout": "command-ref/opt-common.html#opt-timeout",
+    "sec-common-options": "command-ref/opt-common.html",
+    "ch-utilities": "command-ref/utilities.html",
+    "chap-hacking": "contributing/hacking.html",
+    "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes",
+    "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences",
+    "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites",
+    "adv-attr-disallowedReferences": "language/advanced-attributes.html#adv-attr-disallowedReferences",
+    "adv-attr-disallowedRequisites": "language/advanced-attributes.html#adv-attr-disallowedRequisites",
+    "adv-attr-exportReferencesGraph": "language/advanced-attributes.html#adv-attr-exportReferencesGraph",
+    "adv-attr-impureEnvVars": "language/advanced-attributes.html#adv-attr-impureEnvVars",
+    "adv-attr-outputHash": "language/advanced-attributes.html#adv-attr-outputHash",
+    "adv-attr-outputHashAlgo": "language/advanced-attributes.html#adv-attr-outputHashAlgo",
+    "adv-attr-outputHashMode": "language/advanced-attributes.html#adv-attr-outputHashMode",
+    "adv-attr-passAsFile": "language/advanced-attributes.html#adv-attr-passAsFile",
+    "adv-attr-preferLocalBuild": "language/advanced-attributes.html#adv-attr-preferLocalBuild",
+    "fixed-output-drvs": "language/advanced-attributes.html#adv-attr-outputHash",
+    "sec-advanced-attributes": "language/advanced-attributes.html",
+    "builtin-abort": "language/builtins.html#builtins-abort",
+    "builtin-add": "language/builtins.html#builtins-add",
+    "builtin-all": "language/builtins.html#builtins-all",
+    "builtin-any": "language/builtins.html#builtins-any",
+    "builtin-attrNames": "language/builtins.html#builtins-attrNames",
+    "builtin-attrValues": "language/builtins.html#builtins-attrValues",
+    "builtin-baseNameOf": "language/builtins.html#builtins-baseNameOf",
+    "builtin-bitAnd": "language/builtins.html#builtins-bitAnd",
+    "builtin-bitOr": "language/builtins.html#builtins-bitOr",
+    "builtin-bitXor": "language/builtins.html#builtins-bitXor",
+    "builtin-builtins": "language/builtins.html#builtins-builtins",
+    "builtin-compareVersions": "language/builtins.html#builtins-compareVersions",
+    "builtin-concatLists": "language/builtins.html#builtins-concatLists",
+    "builtin-concatStringsSep": "language/builtins.html#builtins-concatStringsSep",
+    "builtin-currentSystem": "language/builtins.html#builtins-currentSystem",
+    "builtin-deepSeq": "language/builtins.html#builtins-deepSeq",
+    "builtin-derivation": "language/builtins.html#builtins-derivation",
+    "builtin-dirOf": "language/builtins.html#builtins-dirOf",
+    "builtin-div": "language/builtins.html#builtins-div",
+    "builtin-elem": "language/builtins.html#builtins-elem",
+    "builtin-elemAt": "language/builtins.html#builtins-elemAt",
+    "builtin-fetchGit": "language/builtins.html#builtins-fetchGit",
+    "builtin-fetchTarball": "language/builtins.html#builtins-fetchTarball",
+    "builtin-fetchurl": "language/builtins.html#builtins-fetchurl",
+    "builtin-filterSource": "language/builtins.html#builtins-filterSource",
+    "builtin-foldl-prime": "language/builtins.html#builtins-foldl-prime",
+    "builtin-fromJSON": "language/builtins.html#builtins-fromJSON",
+    "builtin-functionArgs": "language/builtins.html#builtins-functionArgs",
+    "builtin-genList": "language/builtins.html#builtins-genList",
+    "builtin-getAttr": "language/builtins.html#builtins-getAttr",
+    "builtin-getEnv": "language/builtins.html#builtins-getEnv",
+    "builtin-hasAttr": "language/builtins.html#builtins-hasAttr",
+    "builtin-hashFile": "language/builtins.html#builtins-hashFile",
+    "builtin-hashString": "language/builtins.html#builtins-hashString",
+    "builtin-head": "language/builtins.html#builtins-head",
+    "builtin-import": "language/builtins.html#builtins-import",
+    "builtin-intersectAttrs": "language/builtins.html#builtins-intersectAttrs",
+    "builtin-isAttrs": "language/builtins.html#builtins-isAttrs",
+    "builtin-isBool": "language/builtins.html#builtins-isBool",
+    "builtin-isFloat": "language/builtins.html#builtins-isFloat",
+    "builtin-isFunction": "language/builtins.html#builtins-isFunction",
+    "builtin-isInt": "language/builtins.html#builtins-isInt",
+    "builtin-isList": "language/builtins.html#builtins-isList",
+    "builtin-isNull": "language/builtins.html#builtins-isNull",
+    "builtin-isString": "language/builtins.html#builtins-isString",
+    "builtin-length": "language/builtins.html#builtins-length",
+    "builtin-lessThan": "language/builtins.html#builtins-lessThan",
+    "builtin-listToAttrs": "language/builtins.html#builtins-listToAttrs",
+    "builtin-map": "language/builtins.html#builtins-map",
+    "builtin-match": "language/builtins.html#builtins-match",
+    "builtin-mul": "language/builtins.html#builtins-mul",
+    "builtin-parseDrvName": "language/builtins.html#builtins-parseDrvName",
+    "builtin-path": "language/builtins.html#builtins-path",
+    "builtin-pathExists": "language/builtins.html#builtins-pathExists",
+    "builtin-placeholder": "language/builtins.html#builtins-placeholder",
+    "builtin-readDir": "language/builtins.html#builtins-readDir",
+    "builtin-readFile": "language/builtins.html#builtins-readFile",
+    "builtin-removeAttrs": "language/builtins.html#builtins-removeAttrs",
+    "builtin-replaceStrings": "language/builtins.html#builtins-replaceStrings",
+    "builtin-seq": "language/builtins.html#builtins-seq",
+    "builtin-sort": "language/builtins.html#builtins-sort",
+    "builtin-split": "language/builtins.html#builtins-split",
+    "builtin-splitVersion": "language/builtins.html#builtins-splitVersion",
+    "builtin-stringLength": "language/builtins.html#builtins-stringLength",
+    "builtin-sub": "language/builtins.html#builtins-sub",
+    "builtin-substring": "language/builtins.html#builtins-substring",
+    "builtin-tail": "language/builtins.html#builtins-tail",
+    "builtin-throw": "language/builtins.html#builtins-throw",
+    "builtin-toFile": "language/builtins.html#builtins-toFile",
+    "builtin-toJSON": "language/builtins.html#builtins-toJSON",
+    "builtin-toPath": "language/builtins.html#builtins-toPath",
+    "builtin-toString": "language/builtins.html#builtins-toString",
+    "builtin-toXML": "language/builtins.html#builtins-toXML",
+    "builtin-trace": "language/builtins.html#builtins-trace",
+    "builtin-tryEval": "language/builtins.html#builtins-tryEval",
+    "builtin-typeOf": "language/builtins.html#builtins-typeOf",
+    "ssec-builtins": "language/builtins.html",
+    "attr-system": "language/derivations.html#attr-system",
+    "ssec-derivation": "language/derivations.html",
+    "ch-expression-language": "language/index.html",
+    "sec-constructs": "language/constructs.html",
+    "sect-let-language": "language/constructs.html#let-language",
+    "ss-functions": "language/constructs.html#functions",
+    "sec-language-operators": "language/operators.html",
+    "table-operators": "language/operators.html",
+    "ssec-values": "language/values.html",
+    "gloss-closure": "glossary.html#gloss-closure",
+    "gloss-derivation": "glossary.html#gloss-derivation",
+    "gloss-deriver": "glossary.html#gloss-deriver",
+    "gloss-nar": "glossary.html#gloss-nar",
+    "gloss-output-path": "glossary.html#gloss-output-path",
+    "gloss-profile": "glossary.html#gloss-profile",
+    "gloss-reachable": "glossary.html#gloss-reachable",
+    "gloss-reference": "glossary.html#gloss-reference",
+    "gloss-substitute": "glossary.html#gloss-substitute",
+    "gloss-user-env": "glossary.html#gloss-user-env",
+    "gloss-validity": "glossary.html#gloss-validity",
+    "part-glossary": "glossary.html",
+    "sec-building-source": "installation/building-source.html",
+    "ch-env-variables": "installation/env-variables.html",
+    "sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables",
+    "sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file",
+    "sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file-with-macos-and-the-nix-daemon",
+    "chap-installation": "installation/installation.html",
+    "ch-installing-binary": "installation/installing-binary.html",
+    "sect-macos-installation": "installation/installing-binary.html#macos-installation",
+    "sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation",
+    "sect-macos-installation-encrypted-volume": "installation/installing-binary.html#macos-installation",
+    "sect-macos-installation-recommended-notes": "installation/installing-binary.html#macos-installation",
+    "sect-macos-installation-symlink": "installation/installing-binary.html#macos-installation",
+    "sect-multi-user-installation": "installation/installing-binary.html#multi-user-installation",
+    "sect-nix-install-binary-tarball": "installation/installing-binary.html#installing-from-a-binary-tarball",
+    "sect-nix-install-pinned-version-url": "installation/installing-binary.html#installing-a-pinned-nix-version-from-a-url",
+    "sect-single-user-installation": "installation/installing-binary.html#single-user-installation",
+    "ch-installing-source": "installation/installing-source.html",
+    "ssec-multi-user": "installation/multi-user.html",
+    "ch-nix-security": "installation/nix-security.html",
+    "sec-obtaining-source": "installation/obtaining-source.html",
+    "sec-prerequisites-source": "installation/prerequisites-source.html",
+    "sec-single-user": "installation/single-user.html",
+    "ch-supported-platforms": "installation/supported-platforms.html",
+    "ch-upgrading-nix": "installation/upgrading.html",
+    "ch-about-nix": "introduction.html",
+    "chap-introduction": "introduction.html",
+    "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html",
+    "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html",
+    "sec-channels": "package-management/channels.html",
+    "ssec-copy-closure": "package-management/copy-closure.html",
+    "sec-garbage-collection": "package-management/garbage-collection.html",
+    "ssec-gc-roots": "package-management/garbage-collector-roots.html",
+    "chap-package-management": "package-management/package-management.html",
+    "sec-profiles": "package-management/profiles.html",
+    "ssec-s3-substituter": "package-management/s3-substituter.html",
+    "ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
+    "ssec-s3-substituter-authenticated-reads": "package-management/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
+    "ssec-s3-substituter-authenticated-writes": "package-management/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
+    "sec-sharing-packages": "package-management/sharing-packages.html",
+    "ssec-ssh-substituter": "package-management/ssh-substituter.html",
+    "chap-quick-start": "quick-start.html",
+    "sec-relnotes": "release-notes/release-notes.html",
+    "ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html",
+    "ch-relnotes-0.10": "release-notes/rl-0.10.html",
+    "ssec-relnotes-0.11": "release-notes/rl-0.11.html",
+    "ssec-relnotes-0.12": "release-notes/rl-0.12.html",
+    "ssec-relnotes-0.13": "release-notes/rl-0.13.html",
+    "ssec-relnotes-0.14": "release-notes/rl-0.14.html",
+    "ssec-relnotes-0.15": "release-notes/rl-0.15.html",
+    "ssec-relnotes-0.16": "release-notes/rl-0.16.html",
+    "ch-relnotes-0.5": "release-notes/rl-0.5.html",
+    "ch-relnotes-0.6": "release-notes/rl-0.6.html",
+    "ch-relnotes-0.7": "release-notes/rl-0.7.html",
+    "ch-relnotes-0.8.1": "release-notes/rl-0.8.1.html",
+    "ch-relnotes-0.8": "release-notes/rl-0.8.html",
+    "ch-relnotes-0.9.1": "release-notes/rl-0.9.1.html",
+    "ch-relnotes-0.9.2": "release-notes/rl-0.9.2.html",
+    "ch-relnotes-0.9": "release-notes/rl-0.9.html",
+    "ssec-relnotes-1.0": "release-notes/rl-1.0.html",
+    "ssec-relnotes-1.1": "release-notes/rl-1.1.html",
+    "ssec-relnotes-1.10": "release-notes/rl-1.10.html",
+    "ssec-relnotes-1.11.10": "release-notes/rl-1.11.10.html",
+    "ssec-relnotes-1.11": "release-notes/rl-1.11.html",
+    "ssec-relnotes-1.2": "release-notes/rl-1.2.html",
+    "ssec-relnotes-1.3": "release-notes/rl-1.3.html",
+    "ssec-relnotes-1.4": "release-notes/rl-1.4.html",
+    "ssec-relnotes-1.5.1": "release-notes/rl-1.5.1.html",
+    "ssec-relnotes-1.5.2": "release-notes/rl-1.5.2.html",
+    "ssec-relnotes-1.5": "release-notes/rl-1.5.html",
+    "ssec-relnotes-1.6.1": "release-notes/rl-1.6.1.html",
+    "ssec-relnotes-1.6.0": "release-notes/rl-1.6.html",
+    "ssec-relnotes-1.7": "release-notes/rl-1.7.html",
+    "ssec-relnotes-1.8": "release-notes/rl-1.8.html",
+    "ssec-relnotes-1.9": "release-notes/rl-1.9.html",
+    "ssec-relnotes-2.0": "release-notes/rl-2.0.html",
+    "ssec-relnotes-2.1": "release-notes/rl-2.1.html",
+    "ssec-relnotes-2.2": "release-notes/rl-2.2.html",
+    "ssec-relnotes-2.3": "release-notes/rl-2.3.html"
+  },
+  "language/values": {
+    "simple-values": "#primitives",
+    "lists": "#list",
+    "strings": "#string",
+    "lists": "#list",
+    "attribute-sets": "#attribute-set"
+  }
 };
 
-var isRoot = (document.location.pathname.endsWith('/') || document.location.pathname.endsWith('/index.html')) && path_to_root === '';
-if (isRoot && redirects[document.location.hash]) {
-  document.location.href = path_to_root + redirects[document.location.hash];
+
+function pathsMatch(a, b, path_to_root) {
+  // Do paths `a` and `b` match?
+  //
+  // This is more involved than it should be:
+  //
+  // 1. Path `b` can have an have an arbitrary prefix.
+  //
+  // 2. `path_to_root` consists only of `../`s and determines the depth
+  //     of `b` relative to the prefix:
+  //
+  //          `document.location.pathname`
+  //        |-----------------------------|
+  //        <prefix>/<path>/[<file>[.html]][#<anchor>]
+  //                 |----|
+  //             `path_to_root` has same number of segments
+  //
+  // 3. The following paths are equivalent:
+  //
+  //        foo/bar/
+  //        foo/bar/index.html
+  //        foo/bar/index
+  //
+  //  4. The following paths are also equivalent:
+  //
+  //         foo/bar/baz
+  //         foo/bar/baz.html
+  //
+  // We can use `path_to_root` to discern prefix from path.
+  //
+  // The last element of the following split is always empty.
+  // Example: '../../'.split('/') -> [ '..', '..', '' ]
+  const depth = path_to_root.split('/').length - 1;
+  var segmentsB = b.split('/');
+  // get file name of `b`
+  const fileB = segmentsB.pop(); // mutates `segmentsB`!
+  // get path of `b` without prefix and file name
+  const pathB = segmentsB.slice(segmentsB.length - depth).join('/');
+
+  var segmentsA = a.split('/');
+  const fileA = segmentsA.pop(); // mutates `segmentsA`!
+  const pathA = segmentsA.join('/')
+
+  function normalize(file) {
+    if (file === '') { return "index.html"; }
+    if (!file.endsWith('.html')) { return file + '.html'; }
+    return file;
+  }
+
+  return pathA === pathB && normalize(fileA) === normalize(fileB);
+}
+
+// The anchor starts with the hash character (`#`),
+// but our redirect declarations don't, so we strip it.
+// Example: document.location.hash -> '#foo'
+const anchor = document.location.hash.substring(1);
+
+for (const [path, redirect] of Object.entries(redirects)) {
+  // The global variable `path_to_root` is set by `mdBook`:
+  //
+  // > This is a path containing exclusively `../`'s that points to the root of the
+  // > book from the current file. Since the original directory structure is
+  // > maintained, it is useful to prepend relative links with this `path_to_root`.
+  //
+  // Source: https://phaiax.github.io/mdBook/format/theme/index-hbs.html#data
+  if (pathsMatch(path, document.location.pathname, path_to_root) && redirect[anchor]) {
+    document.location.href = redirect[anchor];
+    break;
+  }
 }

From 548c904d4007bbf6d03ebe06d700af0b96e976f1 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 8 Sep 2022 11:57:49 +0200
Subject: [PATCH 029/112] optimize performance

remove loops and function calls, modify arrays in place

this makes the whole thing harder to read, and probably only
marginally faster.
---
 doc/manual/redirects.js | 141 +++++++++++++++++++++-------------------
 1 file changed, 75 insertions(+), 66 deletions(-)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index 45fbfffab..af3fc8782 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -1,9 +1,12 @@
-// Redirect rules for anchors ensure backwards compatibility of URLs.
-// This must be done on the client side, as web servers do not see the anchor part of the URL.
+// redirect rules for anchors ensure backwards compatibility of URLs.
+// this must be done on the client side, as web servers do not see the anchor part of the URL.
 
-// Redirections are declared as follows:
-// Each entry has as key the matched URL path relative to the mdBook document root.
-// Each entry is a set of key-value pairs, where
+// redirections are declared as follows:
+// each entry has as key the matched URL path relative to the mdBook document root.
+//
+//     IMPORTANT: it must specify the full path with file name and suffix
+//
+// each entry is a set of key-value pairs, where
 // - keys are anchors on to the matched path.
 // - values are redirection targets relative to the current path.
 
@@ -332,7 +335,7 @@ var redirects = {
     "ssec-relnotes-2.2": "release-notes/rl-2.2.html",
     "ssec-relnotes-2.3": "release-notes/rl-2.3.html"
   },
-  "language/values": {
+  "language/values.html": {
     "simple-values": "#primitives",
     "lists": "#list",
     "strings": "#string",
@@ -341,73 +344,79 @@ var redirects = {
   }
 };
 
+// the following code matches the current page's URL against the set of redirects.
+//
+// it is written to minimize the latency between page load and redirect.
+// therefore we avoid function calls, copying data, and unnecessary loops.
+// IMPORTANT: we use stateful array operations and their order matters!
+//
+// matching URLs is more involved than it should be:
+//
+// 1. `document.location.pathname` can have an have an arbitrary prefix.
+//
+// 2. `path_to_root` is set by mdBook and consists only of `../`s and
+//    determines the depth of `<path>` relative to the prefix:
+//
+//          `document.location.pathname`
+//        |------------------------------|
+//        /<prefix>/<path>/[<file>[.html]][#<anchor>]
+//                  |----|
+//              `path_to_root` has same number of segments
+//
+//    source: https://phaiax.github.io/mdBook/format/theme/index-hbs.html#data
+//
+// 3. the following paths are equivalent:
+//
+//        /foo/bar/
+//        /foo/bar/index.html
+//        /foo/bar/index
+//
+//  4. the following paths are also equivalent:
+//
+//        /foo/bar/baz
+//        /foo/bar/baz.html
+//
 
-function pathsMatch(a, b, path_to_root) {
-  // Do paths `a` and `b` match?
-  //
-  // This is more involved than it should be:
-  //
-  // 1. Path `b` can have an have an arbitrary prefix.
-  //
-  // 2. `path_to_root` consists only of `../`s and determines the depth
-  //     of `b` relative to the prefix:
-  //
-  //          `document.location.pathname`
-  //        |-----------------------------|
-  //        <prefix>/<path>/[<file>[.html]][#<anchor>]
-  //                 |----|
-  //             `path_to_root` has same number of segments
-  //
-  // 3. The following paths are equivalent:
-  //
-  //        foo/bar/
-  //        foo/bar/index.html
-  //        foo/bar/index
-  //
-  //  4. The following paths are also equivalent:
-  //
-  //         foo/bar/baz
-  //         foo/bar/baz.html
-  //
-  // We can use `path_to_root` to discern prefix from path.
-  //
-  // The last element of the following split is always empty.
-  // Example: '../../'.split('/') -> [ '..', '..', '' ]
-  const depth = path_to_root.split('/').length - 1;
-  var segmentsB = b.split('/');
-  // get file name of `b`
-  const fileB = segmentsB.pop(); // mutates `segmentsB`!
-  // get path of `b` without prefix and file name
-  const pathB = segmentsB.slice(segmentsB.length - depth).join('/');
+var segments = document.location.pathname.split('/');
 
-  var segmentsA = a.split('/');
-  const fileA = segmentsA.pop(); // mutates `segmentsA`!
-  const pathA = segmentsA.join('/')
+var file = segments.pop();
 
-  function normalize(file) {
-    if (file === '') { return "index.html"; }
-    if (!file.endsWith('.html')) { return file + '.html'; }
-    return file;
-  }
+// normalize file name
+if (file === '') { file = "index.html"; }
+else if (!file.endsWith('.html')) { file = file + '.html'; }
 
-  return pathA === pathB && normalize(fileA) === normalize(fileB);
-}
+segments.push(file);
 
-// The anchor starts with the hash character (`#`),
+// use `path_to_root` to discern prefix from path.
+const depth = path_to_root.split('/').length;
+
+// remove segments containing prefix. the following works because
+// 1. the original `document.location.pathname` is absolute,
+//    hence first element of `segments` is always empty.
+// 2. last element of splitting `path_to_root` is also always empty.
+// 3. last element of `segments` is the file name.
+//
+// visual example:
+//
+//     '/foo/bar/baz.html'.split('/') -> [ '', 'foo', 'bar', 'baz.html' ]
+//          '../'.split('/')          -> [ '..', '' ]
+//
+// the following operations will then result in
+//
+//     path = 'bar/baz.html'
+//
+segments.splice(0, segments.length - depth);
+const path = segments.join('/');
+
+// anchor starts with the hash character (`#`),
 // but our redirect declarations don't, so we strip it.
-// Example: document.location.hash -> '#foo'
+// example: document.location.hash -> '#foo'
 const anchor = document.location.hash.substring(1);
 
-for (const [path, redirect] of Object.entries(redirects)) {
-  // The global variable `path_to_root` is set by `mdBook`:
-  //
-  // > This is a path containing exclusively `../`'s that points to the root of the
-  // > book from the current file. Since the original directory structure is
-  // > maintained, it is useful to prepend relative links with this `path_to_root`.
-  //
-  // Source: https://phaiax.github.io/mdBook/format/theme/index-hbs.html#data
-  if (pathsMatch(path, document.location.pathname, path_to_root) && redirect[anchor]) {
-    document.location.href = redirect[anchor];
-    break;
+const redirect = redirects[path];
+if (redirect) {
+  const target = redirect[anchor];
+  if (target) {
+    document.location.href = target;
   }
 }

From 8dd5ba2f472172eb1a8a8df31715726cc53d6344 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 9 Sep 2022 09:54:24 +0200
Subject: [PATCH 030/112] more precise variable types

---
 doc/manual/redirects.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index af3fc8782..d9b27d866 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -10,7 +10,7 @@
 // - keys are anchors on to the matched path.
 // - values are redirection targets relative to the current path.
 
-var redirects = {
+const redirects = {
  "index.html": {
     "part-advanced-topics": "advanced-topics/advanced-topics.html",
     "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html",
@@ -377,9 +377,9 @@ var redirects = {
 //        /foo/bar/baz.html
 //
 
-var segments = document.location.pathname.split('/');
+let segments = document.location.pathname.split('/');
 
-var file = segments.pop();
+let file = segments.pop();
 
 // normalize file name
 if (file === '') { file = "index.html"; }

From d9abce4ad4b6888183271c0a4051981dee5fffe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 12 Sep 2022 22:50:18 +0200
Subject: [PATCH 031/112] libfetchers: avoid api.github.com ratelimit if no
 github token is set

If we don't have any github token, we won't be able to fetch private
repos, but we are also more likely to run into API limits since
we don't have a token. To mitigate this only ever use the github api
if we actually have a token.
---
 src/libfetchers/github.cc | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index a491d82a6..2115ce2f5 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -262,17 +262,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
     DownloadUrl getDownloadUrl(const Input & input) const override
     {
-        // FIXME: use regular /archive URLs instead? api.github.com
-        // might have stricter rate limits.
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt(
-            host == "github.com"
-            ? "https://api.%s/repos/%s/%s/tarball/%s"
-            : "https://%s/api/v3/repos/%s/%s/tarball/%s",
-            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+        Headers headers = makeHeadersWithAuthTokens(host);
+        // If we have no auth headers then we default to the public archive
+        // urls so we do not run into rate limits.
+        const auto urlFmt =
+            host != "github.com"
+            ? "https://%s/api/v3/repos/%s/%s/tarball/%s"
+            : headers.empty()
+            ? "https://%s/%s/%s/archive/%s.tar.gz"
+            : "https://api.%s/repos/%s/%s/tarball/%s";
+
+        const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(host);
         return DownloadUrl { url, headers };
     }
 

From c7059c9b24b803a0ae039650de5c9103afb46c2d Mon Sep 17 00:00:00 2001
From: Andrew Brooks <andrew.brooks@flightaware.com>
Date: Fri, 14 Oct 2022 17:27:13 -0500
Subject: [PATCH 032/112] Add test for issue 7146

---
 tests/fetchGit.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 166bccfc7..4ceba0293 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -24,12 +24,14 @@ touch $repo/.gitignore
 git -C $repo add hello .gitignore
 git -C $repo commit -m 'Bla1'
 rev1=$(git -C $repo rev-parse HEAD)
+git -C $repo tag -a tag1 -m tag1
 
 echo world > $repo/hello
 git -C $repo commit -m 'Bla2' -a
 git -C $repo worktree add $TEST_ROOT/worktree
 echo hello >> $TEST_ROOT/worktree/hello
 rev2=$(git -C $repo rev-parse HEAD)
+git -C $repo tag -a tag2 -m tag2
 
 # Fetch a worktree
 unset _NIX_FORCE_HTTP
@@ -217,6 +219,16 @@ rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$
 path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; name = \"foo\"; }).outPath")
 [[ $path9 =~ -foo$ ]]
 
+# Specifying a ref without a rev shouldn't pick a cached rev for a different ref
+export _NIX_FORCE_HTTP=1
+rev_tag1_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag1\"; }).rev")
+rev_tag1=$(git -C $repo rev-parse refs/tags/tag1)
+[[ $rev_tag1_nix = $rev_tag1 ]]
+rev_tag2_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag2\"; }).rev")
+rev_tag2=$(git -C $repo rev-parse refs/tags/tag2)
+[[ $rev_tag2_nix = $rev_tag2 ]]
+unset _NIX_FORCE_HTTP
+
 # should fail if there is no repo
 rm -rf $repo/.git
 (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")

From a259084c50cdb692a6ba6818bb56ee381e67715b Mon Sep 17 00:00:00 2001
From: Andrew Brooks <andrew.brooks@flightaware.com>
Date: Fri, 14 Oct 2022 18:04:47 -0500
Subject: [PATCH 033/112] Fix #7146

When fetching a non-local git repo by ref (and no rev), don't consider unrelated
cached revs for the same repository.
---
 src/libfetchers/git.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index c1a21e764..7b7a1be35 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -485,6 +485,10 @@ struct GitInputScheme : InputScheme
                 }
                 input.attrs.insert_or_assign("ref", *head);
                 unlockedAttrs.insert_or_assign("ref", *head);
+            } else {
+                if (!input.getRev()) {
+                    unlockedAttrs.insert_or_assign("ref", input.getRef().value());
+                }
             }
 
             if (auto res = getCache()->lookup(store, unlockedAttrs)) {

From 11b5cf95615ffb2a2475ac444f4fe103e471748f Mon Sep 17 00:00:00 2001
From: Patrick Jackson <patrick@jackson.dev>
Date: Tue, 18 Oct 2022 15:09:17 -0700
Subject: [PATCH 034/112] fix manual defaults

---
 src/nix/main.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index d78312944..f008ea2b7 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -369,6 +369,13 @@ void mainWrapped(int argc, char * * argv)
         && args.command->first != "upgrade-nix")
         settings.requireExperimentalFeature(Xp::NixCommand);
 
+    if (args.command->first == "show-config") {
+        // show-config must run before any settings are modified so that it outputs the defaults
+        args.command->second->prepare();
+        args.command->second->run();
+        return;
+    }
+
     if (args.useNet && !haveInternet()) {
         warn("you don't have Internet access; disabling some network-dependent features");
         args.useNet = false;

From 0b64bac65c7a476797d6238631d693f48a0eecc9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 19 Oct 2022 08:50:25 +0000
Subject: [PATCH 035/112] Bump cachix/cachix-action from 10 to 11

Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 10 to 11.
- [Release notes](https://github.com/cachix/cachix-action/releases)
- [Commits](https://github.com/cachix/cachix-action/compare/v10...v11)

---
updated-dependencies:
- dependency-name: cachix/cachix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 505717921..7efb90913 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -21,7 +21,7 @@ jobs:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v18
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v10
+    - uses: cachix/cachix-action@v11
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -59,7 +59,7 @@ jobs:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/install-nix-action@v18
-    - uses: cachix/cachix-action@v10
+    - uses: cachix/cachix-action@v11
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -105,7 +105,7 @@ jobs:
     - uses: cachix/install-nix-action@v18
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v10
+    - uses: cachix/cachix-action@v11
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'

From 1ec8810687859e0af96d844da67ab6c6d6e69f9d Mon Sep 17 00:00:00 2001
From: Johan Herland <johan@herland.net>
Date: Wed, 19 Oct 2022 11:50:58 +0200
Subject: [PATCH 036/112] Explain how Nix handles antiquotation of paths

Specifically, explain why Nix does not _re_evaluate paths during a
`nix repl` session. This is a thing that bit me while playing around
with paths and antiquotation in `nix repl` while reading the Nix
language tutorial at https://nix.dev/tutorials/nix-language.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/language/values.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md
index f09400d02..6fc8c0369 100644
--- a/doc/manual/src/language/values.md
+++ b/doc/manual/src/language/values.md
@@ -150,6 +150,20 @@
   recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division
   operation. `./a.${foo}/b.${bar}` is a path.
 
+  When a path appears in an antiquotation, and is thus coerced into a string,
+  the path is first copied into the Nix store and the resulting string is
+  the Nix store path. For instance `"${./foo.txt}" will cause `foo.txt` in
+  the current directory to be copied into the Nix store and result in the
+  string `"/nix/store/<HASH>-foo.txt"`.
+
+  Note that the Nix language assumes that all input files will remain
+  _unchanged_ during the course of the Nix expression evaluation.
+  If you for example antiquote a file path during a `nix repl` session, and
+  then later in the same session, after having changed the file contents,
+  evaluate the antiquotation with the file path again, then Nix will still
+  return the first store path. It will _not_ reread the file contents to
+  produce a different Nix store path.
+
 - <a id="type-boolean" href="#type-boolean">Boolean</a>
 
   *Booleans* with values `true` and `false`.

From c63646b693f6f226e0f3a623b0d09839b08609f5 Mon Sep 17 00:00:00 2001
From: sternenseemann <sternenseemann@systemli.org>
Date: Wed, 19 Oct 2022 18:39:26 +0200
Subject: [PATCH 037/112] tests/lang: re-enable eval-okay-eq.nix tests

This is a really old test case (which was originally written before the
proper Nix syntax). The tested deep comparison behavior was implemented
and reverted soon after due to performance problems, but it has been
restored in today's Nix again (thanks to the derivation comparison
optimization, presumably).
---
 tests/lang/eval-okay-eq.exp          | 1 +
 tests/lang/eval-okay-eq.exp.disabled | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 tests/lang/eval-okay-eq.exp
 delete mode 100644 tests/lang/eval-okay-eq.exp.disabled

diff --git a/tests/lang/eval-okay-eq.exp b/tests/lang/eval-okay-eq.exp
new file mode 100644
index 000000000..27ba77dda
--- /dev/null
+++ b/tests/lang/eval-okay-eq.exp
@@ -0,0 +1 @@
+true
diff --git a/tests/lang/eval-okay-eq.exp.disabled b/tests/lang/eval-okay-eq.exp.disabled
deleted file mode 100644
index 2015847b6..000000000
--- a/tests/lang/eval-okay-eq.exp.disabled
+++ /dev/null
@@ -1 +0,0 @@
-Bool(True)

From b44df1370187bea82814a2da50a6ded54fca65f2 Mon Sep 17 00:00:00 2001
From: Patrick Jackson <patrick@jackson.dev>
Date: Thu, 20 Oct 2022 12:05:04 -0700
Subject: [PATCH 038/112] manual: update options generation code to use default
 values

---
 doc/manual/generate-options.nix | 10 +++++-----
 src/nix/main.cc                 |  7 -------
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix
index 2d586fa1b..680b709c8 100644
--- a/doc/manual/generate-options.nix
+++ b/doc/manual/generate-options.nix
@@ -11,16 +11,16 @@ concatStrings (map
     + concatStrings (map (s: "    ${s}\n") (splitLines option.description)) + "\n\n"
     + (if option.documentDefault
        then "    **Default:** " + (
-       if option.value == "" || option.value == []
+       if option.defaultValue == "" || option.defaultValue == []
        then "*empty*"
-       else if isBool option.value
-       then (if option.value then "`true`" else "`false`")
+       else if isBool option.defaultValue
+       then (if option.defaultValue then "`true`" else "`false`")
        else
          # n.b. a StringMap value type is specified as a string, but
          # this shows the value type.  The empty stringmap is "null" in
          # JSON, but that converts to "{ }" here.
-         (if isAttrs option.value then "`\"\"`"
-          else "`" + toString option.value + "`")) + "\n\n"
+         (if isAttrs option.defaultValue then "`\"\"`"
+          else "`" + toString option.defaultValue + "`")) + "\n\n"
        else "    **Default:** *machine-specific*\n")
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
diff --git a/src/nix/main.cc b/src/nix/main.cc
index f008ea2b7..d78312944 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -369,13 +369,6 @@ void mainWrapped(int argc, char * * argv)
         && args.command->first != "upgrade-nix")
         settings.requireExperimentalFeature(Xp::NixCommand);
 
-    if (args.command->first == "show-config") {
-        // show-config must run before any settings are modified so that it outputs the defaults
-        args.command->second->prepare();
-        args.command->second->run();
-        return;
-    }
-
     if (args.useNet && !haveInternet()) {
         warn("you don't have Internet access; disabling some network-dependent features");
         args.useNet = false;

From 8e7804273cec1bb3fa05ce09a37f0732b71a72ec Mon Sep 17 00:00:00 2001
From: Austin Kiekintveld <akiekintveld@icloud.com>
Date: Sat, 22 Oct 2022 19:51:22 -0500
Subject: [PATCH 039/112] Defer to SSH config files for ForwardAgent option

Currently, Nix passes `-a` when it runs commands on a remote machine via
SSH, which disables agent forwarding. This causes issues when the
`ForwardAgent` option is set in SSH config files, as the command line
operation always overrides those.

In particular, this causes issues if the command being run is `sudo`
and the remote machine is configured with the equivalent of NixOS's
`security.pam.enableSSHAgentAuth` option. Not allowing SSH agent
forwarding can cause authentication to fail unexpectedly.

This can currently be worked around by setting `NIX_SSHOPTS="-A"`, but
we should defer to the options in the SSH config files to be least
surprising for users.
---
 src/libstore/ssh.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 1bbad71f2..69bfe3418 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -67,7 +67,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         if (fakeSSH) {
             args = { "bash", "-c" };
         } else {
-            args = { "ssh", host.c_str(), "-x", "-a" };
+            args = { "ssh", host.c_str(), "-x" };
             addCommonSSHOpts(args);
             if (socketPath != "")
                 args.insert(args.end(), {"-S", socketPath});

From 334fa81d0828f79d832a029ba16e0c807ec15554 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sun, 23 Oct 2022 06:54:11 -0400
Subject: [PATCH 040/112] Mark flakes with .type = "flake".

Fixes #7186
---
 src/libexpr/flake/call-flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix
index 932ac5e90..87d7a126b 100644
--- a/src/libexpr/flake/call-flake.nix
+++ b/src/libexpr/flake/call-flake.nix
@@ -43,7 +43,7 @@ let
 
           outputs = flake.outputs (inputs // { self = result; });
 
-          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
+          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; type = "flake"; };
         in
           if node.flake or true then
             assert builtins.isFunction flake.outputs;

From a9a868fe6a456a52cf08244dbb87d67798ed3f9c Mon Sep 17 00:00:00 2001
From: Michal Sojka <michal.sojka@cvut.cz>
Date: Mon, 24 Oct 2022 08:49:46 +0200
Subject: [PATCH 041/112] Improve --profile description

The description of the --profile option talks about the "update" operation.
This is probably meant for operations such as "nix profile install", but the
same option is reused in other subcommands, which do not update the profile,
such as "nix profile {list,history,diff-closures}".

We update the description to make sense in both contexts.
---
 src/libcmd/command.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index 1fdd9e0bd..0740ea960 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -226,7 +226,7 @@ MixProfile::MixProfile()
 {
     addFlag({
         .longName = "profile",
-        .description = "The profile to update.",
+        .description = "The profile to operate on.",
         .labels = {"path"},
         .handler = {&profile},
         .completer = completePath

From da2c61637bbf5d22fc2aa3c8d24005f364d28fa2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Oct 2022 16:48:31 +0200
Subject: [PATCH 042/112] Use _type

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libexpr/flake/call-flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix
index 87d7a126b..8061db3df 100644
--- a/src/libexpr/flake/call-flake.nix
+++ b/src/libexpr/flake/call-flake.nix
@@ -43,7 +43,7 @@ let
 
           outputs = flake.outputs (inputs // { self = result; });
 
-          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; type = "flake"; };
+          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; };
         in
           if node.flake or true then
             assert builtins.isFunction flake.outputs;

From a71e3172afc08e4d44ad719c631f731c37bd5491 Mon Sep 17 00:00:00 2001
From: Gabriella Gonzalez <gabriella@mercury.com>
Date: Tue, 25 Oct 2022 16:43:00 -0700
Subject: [PATCH 043/112] Add `--yes` option for Nix installation script

---
 scripts/install-multi-user.sh       | 2 +-
 scripts/install-nix-from-closure.sh | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index a39339050..96c0f302b 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -58,7 +58,7 @@ readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
 
 readonly ROOT_HOME=~root
 
-if [ -t 0 ]; then
+if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then
     readonly IS_HEADLESS='no'
 else
     readonly IS_HEADLESS='yes'
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index d4eed2efe..ccd19e1a8 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -71,6 +71,8 @@ while [ $# -gt 0 ]; do
         #     # intentional tail space
         #     ACTIONS="${ACTIONS}uninstall "
         #     ;;
+        --yes)
+            export NIX_INSTALLER_YES=1;;
         --no-channel-add)
             export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
         --daemon-user-count)
@@ -90,7 +92,7 @@ while [ $# -gt 0 ]; do
             shift;;
         *)
             {
-                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--nix-extra-conf-file FILE]"
+                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--yes] [--no-channel-add] [--no-modify-profile] [--nix-extra-conf-file FILE]"
 
                 echo "Choose installation method."
                 echo ""
@@ -104,6 +106,8 @@ while [ $# -gt 0 ]; do
                 echo "              trivial to uninstall."
                 echo "              (default)"
                 echo ""
+                echo " --yes:               Run the script non-interactively, accepting all prompts."
+                echo ""
                 echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
                 echo ""
                 echo " --no-modify-profile: Don't modify the user profile to automatically load nix."

From aff6d10934f046066ed68b0d54fa077e726008e5 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Wed, 26 Oct 2022 10:05:27 +0200
Subject: [PATCH 044/112] nix run: fix "'defaultApp.x86_64-linux' should have
 type 'derivation'"

---
 src/nix/app.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix/app.cc b/src/nix/app.cc
index 821964f86..48de8fb82 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -66,7 +66,9 @@ UnresolvedApp Installable::toApp(EvalState & state)
 
     auto type = cursor->getAttr("type")->getString();
 
-    std::string expected = !attrPath.empty() && state.symbols[attrPath[0]] == "apps" ? "app" : "derivation";
+    std::string expected = !attrPath.empty() &&
+        (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp")
+        ? "app" : "derivation";
     if (type != expected)
         throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected);
 

From 9bff7e8ee26c97441858e1cea097f44c6db61235 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Tue, 25 Oct 2022 21:17:32 +0200
Subject: [PATCH 045/112] Fix `nix __build-remote`

Because of a wrong index, `nix __build-remote` wasn't working.

Fix the index to restore the command (and the build hook).
---
 src/nix/main.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index d78312944..f8e93e367 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -270,7 +270,7 @@ void mainWrapped(int argc, char * * argv)
     programPath = argv[0];
     auto programName = std::string(baseNameOf(programPath));
 
-    if (argc > 0 && std::string_view(argv[0]) == "__build-remote") {
+    if (argc > 1 && std::string_view(argv[1]) == "__build-remote") {
         programName = "build-remote";
         argv++; argc--;
     }

From f8d01933838f719b2511a9a73a5fa710cdd59496 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Thu, 27 Oct 2022 11:53:04 +0200
Subject: [PATCH 046/112] Pass the right argv when calling the build hook

Call it as `['nix', '__build-remote', ... ]` rather than the previous
`["__build-remote", "nix __build-remote", ... ]` which seemed to have
been most likely unintended
---
 src/libstore/build/hook-instance.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc
index 1f19ddccc..cb58a1f02 100644
--- a/src/libstore/build/hook-instance.cc
+++ b/src/libstore/build/hook-instance.cc
@@ -16,11 +16,11 @@ HookInstance::HookInstance()
     buildHookArgs.pop_front();
 
     Strings args;
+    args.push_back(std::string(baseNameOf(buildHook)));
 
     for (auto & arg : buildHookArgs)
         args.push_back(arg);
 
-    args.push_back(std::string(baseNameOf(settings.buildHook.get())));
     args.push_back(std::to_string(verbosity));
 
     /* Create a pipe to get the output of the child. */

From 48f840cff89490b2a8bf87d03345beb7113eef0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 27 Oct 2022 18:13:15 +0200
Subject: [PATCH 047/112] assign fricklerhandwerk for documentation releated
 PRs

---
 .github/assign-by-files.yml           |  5 +++++
 .github/workflows/assign-reviewer.yml | 12 ++++++++++++
 2 files changed, 17 insertions(+)
 create mode 100644 .github/assign-by-files.yml
 create mode 100644 .github/workflows/assign-reviewer.yml

diff --git a/.github/assign-by-files.yml b/.github/assign-by-files.yml
new file mode 100644
index 000000000..f13b71776
--- /dev/null
+++ b/.github/assign-by-files.yml
@@ -0,0 +1,5 @@
+---
+# This files is used by https://github.com/marketplace/actions/auto-assign-reviewer-by-files
+# to assign maintainers
+"doc/**/*":
+  - fricklerhandwerk
diff --git a/.github/workflows/assign-reviewer.yml b/.github/workflows/assign-reviewer.yml
new file mode 100644
index 000000000..4371cbff4
--- /dev/null
+++ b/.github/workflows/assign-reviewer.yml
@@ -0,0 +1,12 @@
+name: "Auto Assign"
+on:
+  - pull_request
+
+jobs:
+  assign_reviewer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: shufo/auto-assign-reviewer-by-files@v1.1.4
+        with:
+          config: ".github/assign-by-files.yml"
+          token: ${{ secrets.GITHUB_TOKEN }}

From cd86eeb693342c79b01ff38ca723088dc1e42291 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= <jfroche@affinitic.be>
Date: Fri, 28 Oct 2022 12:19:37 +0200
Subject: [PATCH 048/112] Move savedArgv into libmain

`savedArgv` is not accessible by plugins when defined in main binary.
Moving it into one of the nix lib fix the problem.
---
 src/libmain/shared.cc | 1 +
 src/nix/main.cc       | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index c1cf38565..a58428762 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -33,6 +33,7 @@
 
 namespace nix {
 
+char * * savedArgv;
 
 static bool gcWarning = true;
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index d78312944..956730276 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -53,7 +53,6 @@ static bool haveInternet()
 }
 
 std::string programPath;
-char * * savedArgv;
 
 struct HelpRequested { };
 

From 9f1dd0df5b54a7dc75b618034482ed42ce34383d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Sat, 29 Oct 2022 21:51:29 +0200
Subject: [PATCH 049/112] Update test after api.github.com ratelimit avoidance

---
 tests/github-flakes.nix | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix
index fc481c7e3..1b60a9f9a 100644
--- a/tests/github-flakes.nix
+++ b/tests/github-flakes.nix
@@ -7,7 +7,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") {
 
 let
 
-  # Generate a fake root CA and a fake api.github.com / channels.nixos.org certificate.
+  # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate.
   cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; }
     ''
       mkdir -p $out
@@ -18,7 +18,7 @@ let
 
       openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \
         -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr
-      openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:channels.nixos.org") \
+      openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:channels.nixos.org") \
         -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt
     '';
 
@@ -47,18 +47,20 @@ let
 
   api = pkgs.runCommand "nixpkgs-flake" {}
     ''
-      mkdir -p $out/tarball
+      mkdir -p $out/commits
+      echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
+    '';
+
+  archive = pkgs.runCommand "nixpkgs-flake" {}
+    ''
+      mkdir -p $out/archive
 
       dir=NixOS-nixpkgs-${nixpkgs.shortRev}
       cp -prd ${nixpkgs} $dir
       # Set the correct timestamp in the tarball.
       find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} --
-      tar cfz $out/tarball/${nixpkgs.rev} $dir --hard-dereference
-
-      mkdir -p $out/commits
-      echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
+      tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference
     '';
-
 in
 
 makeTest (
@@ -97,6 +99,16 @@ makeTest (
                   }
                 ];
             };
+          services.httpd.virtualHosts."github.com" =
+            { forceSSL = true;
+              sslServerKey = "${cert}/server.key";
+              sslServerCert = "${cert}/server.crt";
+              servedDirs =
+                [ { urlPath = "/NixOS/nixpkgs";
+                    dir = archive;
+                  }
+                ];
+            };
         };
 
       client =
@@ -109,7 +121,7 @@ makeTest (
           nix.extraOptions = "experimental-features = nix-command flakes";
           environment.systemPackages = [ pkgs.jq ];
           networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} =
-            [ "channels.nixos.org" "api.github.com" ];
+            [ "channels.nixos.org" "api.github.com" "github.com" ];
           security.pki.certificateFiles = [ "${cert}/ca.crt" ];
         };
     };
@@ -123,7 +135,7 @@ makeTest (
 
     github.wait_for_unit("httpd.service")
 
-    client.succeed("curl -v https://api.github.com/ >&2")
+    client.succeed("curl -v https://github.com/ >&2")
     client.succeed("nix registry list | grep nixpkgs")
 
     rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision")

From e00761af73f68fa7e6b833b4800a17e93f715097 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sun, 30 Oct 2022 07:10:58 +0100
Subject: [PATCH 050/112] Also test github flakes when access tokens are
 provided

---
 tests/github-flakes.nix | 66 +++++++++++++++++++++++++++++++++++------
 1 file changed, 57 insertions(+), 9 deletions(-)

diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix
index 1b60a9f9a..43a4f1432 100644
--- a/tests/github-flakes.nix
+++ b/tests/github-flakes.nix
@@ -37,6 +37,17 @@ let
               "owner": "NixOS",
               "repo": "nixpkgs"
             }
+          },
+          {
+            "from": {
+              "type": "indirect",
+              "id": "private-flake"
+            },
+            "to": {
+              "type": "github",
+              "owner": "fancy-enterprise",
+              "repo": "private-flake"
+            }
           }
         ],
         "version": 2
@@ -45,9 +56,27 @@ let
     destination = "/flake-registry.json";
   };
 
-  api = pkgs.runCommand "nixpkgs-flake" {}
+  private-flake-rev = "9f1dd0df5b54a7dc75b618034482ed42ce34383d";
+
+  private-flake-api = pkgs.runCommand "private-flake" {}
+    ''
+      mkdir -p $out/{commits,tarball}
+
+      # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit
+      echo '{"sha": "${private-flake-rev}"}' > $out/commits/HEAD
+
+      # Setup tarball download via API
+      dir=private-flake
+      mkdir $dir
+      echo '{ outputs = {...}: {}; }' > $dir/flake.nix
+      tar cfz $out/tarball/${private-flake-rev} $dir --hard-dereference
+    '';
+
+  nixpkgs-api = pkgs.runCommand "nixpkgs-flake" {}
     ''
       mkdir -p $out/commits
+
+      # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit
       echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
     '';
 
@@ -95,7 +124,10 @@ makeTest (
               sslServerCert = "${cert}/server.crt";
               servedDirs =
                 [ { urlPath = "/repos/NixOS/nixpkgs";
-                    dir = api;
+                    dir = nixpkgs-api;
+                  }
+                  { urlPath = "/repos/fancy-enterprise/private-flake";
+                    dir = private-flake-api;
                   }
                 ];
             };
@@ -119,7 +151,6 @@ makeTest (
           virtualisation.memorySize = 4096;
           nix.binaryCaches = lib.mkForce [ ];
           nix.extraOptions = "experimental-features = nix-command flakes";
-          environment.systemPackages = [ pkgs.jq ];
           networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} =
             [ "channels.nixos.org" "api.github.com" "github.com" ];
           security.pki.certificateFiles = [ "${cert}/ca.crt" ];
@@ -133,22 +164,39 @@ makeTest (
 
     start_all()
 
+    def cat_log():
+         github.succeed("cat /var/log/httpd/*.log >&2")
+
     github.wait_for_unit("httpd.service")
 
     client.succeed("curl -v https://github.com/ >&2")
-    client.succeed("nix registry list | grep nixpkgs")
+    out = client.succeed("nix registry list")
+    print(out)
+    assert "github:NixOS/nixpkgs" in out, "nixpkgs flake not found"
+    assert "github:fancy-enterprise/private-flake" in out, "private flake not found"
+    cat_log()
 
-    rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision")
-    assert rev.strip() == "${nixpkgs.rev}", "revision mismatch"
+    # If no github access token is provided, nix should use the public archive url...
+    out = client.succeed("nix flake metadata nixpkgs --json")
+    print(out)
+    info = json.loads(out)
+    assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}"
+    cat_log()
+
+    # ... otherwise it should use the API
+    out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")
+    print(out)
+    info = json.loads(out)
+    assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}"
+    cat_log()
 
     client.succeed("nix registry pin nixpkgs")
-
-    client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2")
+    client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2")
 
     # Shut down the web server. The flake should be cached on the client.
     github.succeed("systemctl stop httpd.service")
 
-    info = json.loads(client.succeed("nix flake info nixpkgs --json"))
+    info = json.loads(client.succeed("nix flake metadata nixpkgs --json"))
     date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified']))
     assert date == "${nixpkgs.lastModifiedDate}", "time mismatch"
 

From 06a6a7959eaa859dab591bb5386031c0d3d1b803 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 31 Oct 2022 22:08:02 +0000
Subject: [PATCH 051/112] Bump cachix/cachix-action from 11 to 12

Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 11 to 12.
- [Release notes](https://github.com/cachix/cachix-action/releases)
- [Commits](https://github.com/cachix/cachix-action/compare/v11...v12)

---
updated-dependencies:
- dependency-name: cachix/cachix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7efb90913..dafba6d85 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -21,7 +21,7 @@ jobs:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v18
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v11
+    - uses: cachix/cachix-action@v12
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -59,7 +59,7 @@ jobs:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/install-nix-action@v18
-    - uses: cachix/cachix-action@v11
+    - uses: cachix/cachix-action@v12
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -105,7 +105,7 @@ jobs:
     - uses: cachix/install-nix-action@v18
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v11
+    - uses: cachix/cachix-action@v12
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'

From 34ea0e2e7b72aa70b4b562eef77c7f3617fed1bb Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Tue, 1 Nov 2022 15:46:30 +0100
Subject: [PATCH 052/112] tarfile: set directory mode to at least 0500, don't
 extract fflags

We don't need SGID, or any ACL's. We also want to keep every dir +rx.
---
 src/libutil/tarfile.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index a7db58559..238d0a7a6 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -77,9 +77,7 @@ TarArchive::~TarArchive()
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
-    int flags = ARCHIVE_EXTRACT_FFLAGS
-        | ARCHIVE_EXTRACT_PERM
-        | ARCHIVE_EXTRACT_TIME
+    int flags = ARCHIVE_EXTRACT_TIME
         | ARCHIVE_EXTRACT_SECURE_SYMLINKS
         | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
 
@@ -98,6 +96,10 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         archive_entry_copy_pathname(entry,
             (destDir + "/" + name).c_str());
 
+        // sources can and do contain dirs with no rx bits
+        if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500)
+            archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500);
+
         // Patch hardlink path
         const char *original_hardlink = archive_entry_hardlink(entry);
         if (original_hardlink) {

From 444af855523b00b0959273bab9d7343312817cb3 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Wed, 5 Oct 2022 12:09:57 +0200
Subject: [PATCH 053/112] Temporarily disable the debugger during completion
 evaluation

readline is not re-entrant, so entering the debugger from the
completioncallback results in an eventual segfault.

The workaround is to temporarily disable the debugger when searching
for possible completions.
---
 src/libcmd/repl.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index df8932087..bb254ff8d 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -384,6 +384,10 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             i++;
         }
     } else {
+        /* Temporarily disable the debugger, to avoid re-entering readline. */
+        auto debug_repl = state->debugRepl;
+        state->debugRepl = nullptr;
+        Finally restoreDebug([&]() { state->debugRepl = debug_repl; });
         try {
             /* This is an expression that should evaluate to an
                attribute set.  Evaluate it to get the names of the

From 16f1720fd2ac1c74043492e71a3e0a3327db4919 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Thu, 3 Nov 2022 10:11:28 +0100
Subject: [PATCH 054/112] SourceExprCommand: swallow EvalError, add tests for
 this

Completing things that would error would print an ugly error in
the middle of your command line. Avoid printing this error.
---
 src/libcmd/installables.cc | 85 ++++++++++++++++++++------------------
 tests/completions.sh       |  6 +++
 2 files changed, 51 insertions(+), 40 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index e097f23b3..e8836c247 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -207,55 +207,60 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 
 void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
-    if (file) {
-        completionType = ctAttrs;
+    try {
+        if (file) {
+            completionType = ctAttrs;
 
-        evalSettings.pureEval = false;
-        auto state = getEvalState();
-        Expr *e = state->parseExprFromFile(
-            resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
-        );
+            evalSettings.pureEval = false;
+            auto state = getEvalState();
+            Expr *e = state->parseExprFromFile(
+                resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
+                );
 
-        Value root;
-        state->eval(e, root);
+            Value root;
+            state->eval(e, root);
 
-        auto autoArgs = getAutoArgs(*state);
+            auto autoArgs = getAutoArgs(*state);
 
-        std::string prefix_ = std::string(prefix);
-        auto sep = prefix_.rfind('.');
-        std::string searchWord;
-        if (sep != std::string::npos) {
-            searchWord = prefix_.substr(sep + 1, std::string::npos);
-            prefix_ = prefix_.substr(0, sep);
-        } else {
-            searchWord = prefix_;
-            prefix_ = "";
-        }
+            std::string prefix_ = std::string(prefix);
+            auto sep = prefix_.rfind('.');
+            std::string searchWord;
+            if (sep != std::string::npos) {
+                searchWord = prefix_.substr(sep + 1, std::string::npos);
+                prefix_ = prefix_.substr(0, sep);
+            } else {
+                searchWord = prefix_;
+                prefix_ = "";
+            }
 
-        auto [v, pos] = findAlongAttrPath(*state, prefix_, *autoArgs, root);
-        Value &v1(*v);
-        state->forceValue(v1, pos);
-        Value v2;
-        state->autoCallFunction(*autoArgs, v1, v2);
+            auto [v, pos] = findAlongAttrPath(*state, prefix_, *autoArgs, root);
+            Value &v1(*v);
+            state->forceValue(v1, pos);
+            Value v2;
+            state->autoCallFunction(*autoArgs, v1, v2);
 
-        if (v2.type() == nAttrs) {
-            for (auto & i : *v2.attrs) {
-                std::string name = state->symbols[i.name];
-                if (name.find(searchWord) == 0) {
-                    if (prefix_ == "")
-                        completions->add(name);
-                    else
-                        completions->add(prefix_ + "." + name);
+            if (v2.type() == nAttrs) {
+                for (auto & i : *v2.attrs) {
+                    std::string name = state->symbols[i.name];
+                    if (name.find(searchWord) == 0) {
+                        if (prefix_ == "")
+                            completions->add(name);
+                        else
+                            completions->add(prefix_ + "." + name);
+                    }
                 }
             }
+        } else {
+            completeFlakeRefWithFragment(
+                getEvalState(),
+                lockFlags,
+                getDefaultFlakeAttrPathPrefixes(),
+                getDefaultFlakeAttrPaths(),
+                prefix);
         }
-    } else {
-        completeFlakeRefWithFragment(
-            getEvalState(),
-            lockFlags,
-            getDefaultFlakeAttrPathPrefixes(),
-            getDefaultFlakeAttrPaths(),
-            prefix);
+    } catch (EvalError& e) {
+        // swallow eval error
+        (void)e;
     }
 }
 
diff --git a/tests/completions.sh b/tests/completions.sh
index 522aa1c86..19dc61098 100644
--- a/tests/completions.sh
+++ b/tests/completions.sh
@@ -28,6 +28,10 @@ cat <<EOF > bar/flake.nix
     };
 }
 EOF
+mkdir -p err
+cat <<EOF > err/flake.nix
+throw "error"
+EOF
 
 # Test the completion of a subcommand
 [[ "$(NIX_GET_COMPLETIONS=1 nix buil)" == $'normal\nbuild\t' ]]
@@ -60,3 +64,5 @@ NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-impo
 # Attr path completions
 [[ "$(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam)" == $'attrs\n./foo#sampleOutput\t' ]]
 [[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp)" == $'attrs\noutputs\t' ]]
+[[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./err/flake.nix outp 2>&1)" == $'attrs' ]]
+[[ "$(NIX_GET_COMPLETIONS=2 nix eval ./err\# 2>&1)" == $'attrs' ]]

From 8e7bbc3c350b5ab1be30f5439e7d05b175b1779d Mon Sep 17 00:00:00 2001
From: Artturin <Artturin@artturin.com>
Date: Thu, 3 Nov 2022 21:53:11 +0200
Subject: [PATCH 055/112] tests/impure-derivations.sh: remove unknown
 experimental feature 'ca-references'

ca-references was stabilized in d589a6aa8a5d0c9f391400d7e0e209106e89c857
---
 tests/impure-derivations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh
index 35ae3f5d3..7ca9ce742 100644
--- a/tests/impure-derivations.sh
+++ b/tests/impure-derivations.sh
@@ -2,7 +2,7 @@ source common.sh
 
 requireDaemonNewerThan "2.8pre20220311"
 
-enableFeatures "ca-derivations ca-references impure-derivations"
+enableFeatures "ca-derivations impure-derivations"
 restartDaemon
 
 set -o pipefail

From dad859ba0f2d1faad5d687348ad1a4e8ebe8b123 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Na=C3=AFm=20Favier?= <n@monade.li>
Date: Fri, 4 Nov 2022 12:41:38 +0100
Subject: [PATCH 056/112] Fix printing of eval errors with two format
 placeholders

---
 src/libexpr/eval.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e3716f217..563f24e48 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -904,7 +904,7 @@ void EvalState::throwEvalError(const char * s, const std::string & s2,
     const std::string & s3)
 {
     debugThrowLastTrace(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[noPos]
     }));
 }
@@ -913,7 +913,7 @@ void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::stri
     const std::string & s3)
 {
     debugThrowLastTrace(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[pos]
     }));
 }
@@ -922,7 +922,7 @@ void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::stri
     const std::string & s3, Env & env, Expr & expr)
 {
     debugThrow(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[pos]
     }), env, expr);
 }

From 6259fd7ea6ac331d1037a150b07c01125e80ed8e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 4 Nov 2022 13:29:39 +0100
Subject: [PATCH 057/112] Fix indentation

---
 src/nix-store/nix-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index eb65e7dde..b59a6d026 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -745,8 +745,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         auto current = sink.finish();
         if (current.first != info->narHash) {
             printError("path '%s' was modified! expected hash '%s', got '%s'",
-                    store->printStorePath(path),
-                    info->narHash.to_string(Base32, true),
+                store->printStorePath(path),
+                info->narHash.to_string(Base32, true),
                 current.first.to_string(Base32, true));
             status = 1;
         }

From 40911d7dec75541a400fe8f556d4c70a7f845fac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 4 Nov 2022 13:30:35 +0100
Subject: [PATCH 058/112] Remove stray tab

---
 src/libutil/experimental-features.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 30d071408..670079019 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -14,8 +14,8 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::NoUrlLiterals, "no-url-literals" },
     { Xp::FetchClosure, "fetch-closure" },
     { Xp::ReplFlake, "repl-flake" },
-	{ Xp::AutoAllocateUids, "auto-allocate-uids" },
-	{ Xp::SystemdCgroup, "systemd-cgroup" },
+    { Xp::AutoAllocateUids, "auto-allocate-uids" },
+    { Xp::SystemdCgroup, "systemd-cgroup" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)

From 907f52c3376ea764e3fd143fe48c0bbb8958ef22 Mon Sep 17 00:00:00 2001
From: Patrick Jackson <patrick@jackson.dev>
Date: Fri, 4 Nov 2022 10:49:44 -0700
Subject: [PATCH 059/112] build-remote: Add brackets to error message

---
 src/build-remote/build-remote.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index ff8ba2724..6b81ecc49 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -186,12 +186,12 @@ static int main_build_remote(int argc, char * * argv)
                         // build the hint template.
                         std::string errorText =
                             "Failed to find a machine for remote build!\n"
-                            "derivation: %s\nrequired (system, features): (%s, %s)";
+                            "derivation: %s\nrequired (system, features): (%s, [%s])";
                         errorText += "\n%s available machines:";
                         errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
 
                         for (unsigned int i = 0; i < machines.size(); ++i)
-                            errorText += "\n(%s, %s, %s, %s)";
+                            errorText += "\n([%s], %s, [%s], [%s])";
 
                         // add the template values.
                         std::string drvstr;

From f7ab93b0685b653c60b86540639a42b03389c90b Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sun, 6 Nov 2022 12:25:21 +0100
Subject: [PATCH 060/112] manual: build action -> build task

after discussing this with multiple people, I'm convinced that "build
task" is more precise: a derivation is not an action, but inert until it
is built. also it's easier to pronounce.

proposal: use "build task" for the generic concept "description of how
to derive new files from the contents of existing files". then it will
be easier to distinguish what we mean by "derivation" (a specific data
structure and Nix language value type) and "store derivation" (a
serialisation of a derivation into a file in the Nix store).
---
 doc/manual/src/command-ref/nix-daemon.md | 4 ++--
 doc/manual/src/command-ref/nix-store.md  | 2 +-
 doc/manual/src/glossary.md               | 6 +++---
 doc/manual/src/introduction.md           | 2 +-
 doc/manual/src/language/derivations.md   | 2 +-
 src/nix/daemon.md                        | 2 +-
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-daemon.md b/doc/manual/src/command-ref/nix-daemon.md
index e91cb01dd..04b483be3 100644
--- a/doc/manual/src/command-ref/nix-daemon.md
+++ b/doc/manual/src/command-ref/nix-daemon.md
@@ -8,6 +8,6 @@
 
 # Description
 
-The Nix daemon is necessary in multi-user Nix installations. It performs
-build actions and other operations on the Nix store on behalf of
+The Nix daemon is necessary in multi-user Nix installations. It runs
+build tasks and other operations on the Nix store on behalf of
 unprivileged users.
diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md
index ecd838e8d..1251888e9 100644
--- a/doc/manual/src/command-ref/nix-store.md
+++ b/doc/manual/src/command-ref/nix-store.md
@@ -71,7 +71,7 @@ paths. Realisation is a somewhat overloaded term:
     outputs are already valid, in which case we are done
     immediately. Otherwise, there may be [substitutes](../glossary.md)
     that produce the outputs (e.g., by downloading them). Finally, the
-    outputs can be produced by performing the build action described
+    outputs can be produced by running the build task described
     by the derivation.
 
   - If the store path is not a derivation, realisation ensures that the
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 70a0eb994..b13709f8a 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -1,7 +1,7 @@
 # Glossary
 
   - [derivation]{#gloss-derivation}\
-    A description of a build action. The result of a derivation is a
+    A description of a build task. The result of a derivation is a
     store object. Derivations are typically specified in Nix expressions
     using the [`derivation` primitive](language/derivations.md). These are
     translated into low-level *store derivations* (implicitly by
@@ -53,8 +53,8 @@
     A file that is an immediate child of the Nix store directory. These
     can be regular files, but also entire directory trees. Store objects
     can be sources (objects copied from outside of the store),
-    derivation outputs (objects produced by running a build action), or
-    derivations (files describing a build action).
+    derivation outputs (objects produced by running a build task), or
+    derivations (files describing a build task).
 
   - [input-addressed store object]{#gloss-input-addressed-store-object}\
     A store object produced by building a
diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md
index d87487a07..b54346db8 100644
--- a/doc/manual/src/introduction.md
+++ b/doc/manual/src/introduction.md
@@ -104,7 +104,7 @@ a currently running program.
 
 Packages are built from _Nix expressions_, which is a simple
 functional language.  A Nix expression describes everything that goes
-into a package build action (a “derivation”): other packages, sources,
+into a package build task (a “derivation”): other packages, sources,
 the build script, environment variables for the build script, etc.
 Nix tries very hard to ensure that Nix expressions are
 _deterministic_: building a Nix expression twice should yield the same
diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md
index 3391ec0d8..043a38191 100644
--- a/doc/manual/src/language/derivations.md
+++ b/doc/manual/src/language/derivations.md
@@ -1,7 +1,7 @@
 # Derivations
 
 The most important built-in function is `derivation`, which is used to
-describe a single derivation (a build action). It takes as input a set,
+describe a single derivation (a build task). It takes as input a set,
 the attributes of which specify the inputs of the build.
 
   - There must be an attribute named [`system`]{#attr-system} whose value must be a
diff --git a/src/nix/daemon.md b/src/nix/daemon.md
index e97016a94..d5cdadf08 100644
--- a/src/nix/daemon.md
+++ b/src/nix/daemon.md
@@ -11,7 +11,7 @@ R""(
 # Description
 
 This command runs the Nix daemon, which is a required component in
-multi-user Nix installations. It performs build actions and other
+multi-user Nix installations. It runs build tasks and other
 operations on the Nix store on behalf of non-root users. Usually you
 don't run the daemon directly; instead it's managed by a service
 management framework such as `systemd`.

From 2fde7e0108d70bcba64ebecc5e5c7ee2863e3446 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Nov 2022 16:03:42 +0100
Subject: [PATCH 061/112] Split auto UID allocation from cgroups

Cgroups are now only used for derivations that require the uid-range
range feature. This allows auto UID allocation even on systems that
don't have cgroups (like macOS).

Also, make things work on modern systems that use cgroups v2 (where
there is a single hierarchy and no "systemd" controller).
---
 src/libstore/build/local-derivation-goal.cc |  19 +--
 src/libstore/build/local-derivation-goal.hh |   7 +-
 src/libstore/cgroup.cc                      |   1 +
 src/libstore/globals.cc                     |   4 +
 src/libstore/globals.hh                     |   6 +-
 src/libstore/lock.cc                        | 149 +++++++++++---------
 src/libstore/lock.hh                        |  24 ++--
 src/libstore/parsed-derivations.cc          |   6 +
 src/libstore/parsed-derivations.hh          |   2 +
 9 files changed, 122 insertions(+), 96 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 64540d262..09da87476 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -160,7 +160,7 @@ void LocalDerivationGoal::tryLocalBuild() {
 
     if (useBuildUsers()) {
         if (!buildUser)
-            buildUser = acquireUserLock();
+            buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1);
 
         if (!buildUser) {
             if (!actLock)
@@ -495,8 +495,8 @@ void LocalDerivationGoal::startBuilder()
         }
     }
 
-    useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range");
     useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("Systemd-cgroup");
+    assert(!useSystemdCgroup);
 
     if (useChroot) {
 
@@ -576,7 +576,8 @@ void LocalDerivationGoal::startBuilder()
 
         printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
 
-        if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1)
+        // FIXME: make this 0700
+        if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
         // FIXME: only make root writable for user namespace builds.
@@ -596,8 +597,8 @@ void LocalDerivationGoal::startBuilder()
         createDirs(chrootRootDir + "/etc");
         chownToBuilder(chrootRootDir + "/etc");
 
-        if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536))
-            throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name);
+        if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536))
+            throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name);
 
         /* Declare the build user's group so that programs get a consistent
            view of the system (e.g., "id -gn"). */
@@ -670,7 +671,7 @@ void LocalDerivationGoal::startBuilder()
         #endif
 #endif
     } else {
-        if (useUidRange)
+        if (parsedDrv->useUidRange())
             throw Error("feature 'uid-range' is only supported in sandboxed builds");
         if (useSystemdCgroup)
             throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds");
@@ -934,12 +935,12 @@ void LocalDerivationGoal::startBuilder()
                the calling user (if build users are disabled). */
             uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
             uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
-            uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1;
+            uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1;
 
             writeFile("/proc/" + std::to_string(pid) + "/uid_map",
                 fmt("%d %d %d", sandboxUid(), hostUid, nrIds));
 
-            if (!useUidRange)
+            if (!buildUser || buildUser->getUIDCount() == 1)
                 writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
 
             writeFile("/proc/" + std::to_string(pid) + "/gid_map",
@@ -1793,7 +1794,7 @@ void LocalDerivationGoal::runChild()
                 throw SysError("mounting /proc");
 
             /* Mount sysfs on /sys. */
-            if (useUidRange) {
+            if (buildUser && buildUser->getUIDCount() != 1) {
                 createDirs(chrootRootDir + "/sys");
                 if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
                     throw SysError("mounting /sys");
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index e6700a383..61b0f9145 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -41,9 +41,6 @@ struct LocalDerivationGoal : public DerivationGoal
 
     Path chrootRootDir;
 
-    /* Whether to give the build more than 1 UID. */
-    bool useUidRange = false;
-
     /* Whether to make the 'systemd' cgroup controller available to
        the build. */
     bool useSystemdCgroup = false;
@@ -99,8 +96,8 @@ struct LocalDerivationGoal : public DerivationGoal
        result. */
     std::map<Path, ValidPathInfo> prevInfos;
 
-    uid_t sandboxUid() { return usingUserNamespace ? (useUidRange ? 0 : 1000) : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ? (useUidRange ? 0 : 100)  : buildUser->getGID(); }
+    uid_t sandboxUid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
+    gid_t sandboxGid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 100  : 0) : buildUser->getGID(); }
 
     const static Path homeDir;
 
diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc
index 5d31609da..56e980be3 100644
--- a/src/libstore/cgroup.cc
+++ b/src/libstore/cgroup.cc
@@ -13,6 +13,7 @@
 
 namespace nix {
 
+// FIXME: obsolete, check for cgroup2
 std::map<std::string, std::string> getCgroups(const Path & cgroupFile)
 {
     std::map<std::string, std::string> cgroups;
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index ff658c428..b7f55cae7 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -130,6 +130,10 @@ StringSet Settings::getDefaultSystemFeatures()
        actually require anything special on the machines. */
     StringSet features{"nixos-test", "benchmark", "big-parallel"};
 
+    #if __linux__
+    features.insert("uid-range");
+    #endif
+
     #if __linux__
     if (access("/dev/kvm", R_OK | W_OK) == 0)
         features.insert("kvm");
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d3e86cc55..be741a830 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -46,6 +46,8 @@ struct PluginFilesSetting : public BaseSetting<Paths>
     void set(const std::string & str, bool append = false) override;
 };
 
+const uint32_t maxIdsPerBuild = 1 << 16;
+
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -279,12 +281,10 @@ public:
     Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids",
         "Whether to allocate UIDs for builders automatically."};
 
-    const uint32_t idsPerBuild = 1 << 16;
-
     Setting<uint32_t> startId{this, 872415232, "start-id",
         "The first UID and GID to use for dynamic ID allocation."};
 
-    Setting<uint32_t> uidCount{this, idsPerBuild * 128, "id-count",
+    Setting<uint32_t> uidCount{this, maxIdsPerBuild * 128, "id-count",
         "The number of UIDs/GIDs to use for dynamic ID allocation."};
     #endif
 
diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index cc3977496..ecc51cebe 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -20,12 +20,8 @@ struct SimpleUserLock : UserLock
         killUser(uid);
     }
 
-    std::pair<uid_t, uid_t> getUIDRange() override
-    {
-        assert(uid);
-        return {uid, uid};
-    }
-
+    uid_t getUID() override { assert(uid); return uid; }
+    uid_t getUIDCount() override { return 1; }
     gid_t getGID() override { assert(gid); return gid; }
 
     std::vector<gid_t> getSupplementaryGIDs() override { return supplementaryGIDs; }
@@ -115,48 +111,65 @@ struct SimpleUserLock : UserLock
     }
 };
 
-#if __linux__
-struct CgroupUserLock : UserLock
+struct AutoUserLock : UserLock
 {
     AutoCloseFD fdUserLock;
-    uid_t uid;
+    uid_t firstUid = 0;
+    uid_t nrIds = 1;
+    #if __linux__
+    std::optional<Path> cgroup;
+    #endif
+
+    ~AutoUserLock()
+    {
+        // Get rid of our cgroup, ignoring errors.
+        if (cgroup) rmdir(cgroup->c_str());
+    }
 
     void kill() override
     {
+        #if __linux__
         if (cgroup) {
+            printError("KILL CGROUP %s", *cgroup);
             destroyCgroup(*cgroup);
-            cgroup.reset();
+            if (mkdir(cgroup->c_str(), 0755) == -1)
+                throw SysError("creating cgroup '%s'", *cgroup);
+        } else
+        #endif
+        {
+            assert(firstUid);
+            printError("KILL USER %d", firstUid);
+            killUser(firstUid);
         }
     }
 
-    std::pair<uid_t, uid_t> getUIDRange() override
-    {
-        assert(uid);
-        return {uid, uid + settings.idsPerBuild - 1};
-    }
+    uid_t getUID() override { assert(firstUid); return firstUid; }
+
+    gid_t getUIDCount() override { return nrIds; }
 
     gid_t getGID() override
     {
         // We use the same GID ranges as for the UIDs.
-        assert(uid);
-        return uid;
+        assert(firstUid);
+        return firstUid;
     }
 
     std::vector<gid_t> getSupplementaryGIDs() override { return {}; }
 
-    static std::unique_ptr<UserLock> acquire()
+    static std::unique_ptr<UserLock> acquire(uid_t nrIds)
     {
         settings.requireExperimentalFeature(Xp::AutoAllocateUids);
         assert(settings.startId > 0);
-        assert(settings.startId % settings.idsPerBuild == 0);
-        assert(settings.uidCount % settings.idsPerBuild == 0);
+        assert(settings.startId % maxIdsPerBuild == 0);
+        assert(settings.uidCount % maxIdsPerBuild == 0);
         assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
+        assert(nrIds <= maxIdsPerBuild);
 
         // FIXME: check whether the id range overlaps any known users
 
         createDirs(settings.nixStateDir + "/userpool2");
 
-        size_t nrSlots = settings.uidCount / settings.idsPerBuild;
+        size_t nrSlots = settings.uidCount / maxIdsPerBuild;
 
         for (size_t i = 0; i < nrSlots; i++) {
             debug("trying user slot '%d'", i);
@@ -170,11 +183,47 @@ struct CgroupUserLock : UserLock
                 throw SysError("opening user lock '%s'", fnUserLock);
 
             if (lockFile(fd.get(), ltWrite, false)) {
-                auto lock = std::make_unique<CgroupUserLock>();
+                auto s = drainFD(fd.get());
+
+                #if __linux__
+                if (s != "") {
+                    /* Kill the old cgroup, to ensure there are no
+                       processes left over from an interrupted build. */
+                    destroyCgroup(s);
+                }
+                #endif
+
+                if (ftruncate(fd.get(), 0) == -1)
+                    throw Error("truncating user lock");
+
+                auto lock = std::make_unique<AutoUserLock>();
                 lock->fdUserLock = std::move(fd);
-                lock->uid = settings.startId + i * settings.idsPerBuild;
-                auto s = drainFD(lock->fdUserLock.get());
-                if (s != "") lock->cgroup = s;
+                lock->firstUid = settings.startId + i * maxIdsPerBuild;
+                lock->nrIds = nrIds;
+
+                if (nrIds > 1) {
+                    auto ourCgroups = getCgroups("/proc/self/cgroup");
+                    auto ourCgroup = ourCgroups[""];
+                    if (ourCgroup == "")
+                        throw Error("cannot determine cgroup name from /proc/self/cgroup");
+
+                    auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup);
+
+                    printError("PARENT CGROUP = %s", ourCgroupPath);
+
+                    if (!pathExists(ourCgroupPath))
+                        throw Error("expected cgroup directory '%s'", ourCgroupPath);
+
+                    lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid);
+
+                    printError("CHILD CGROUP = %s", *lock->cgroup);
+
+                    /* Record the cgroup in the lock file. This ensures that
+                       if we subsequently get executed under a different parent
+                       cgroup, we kill the previous cgroup first. */
+                    writeFull(lock->fdUserLock.get(), *lock->cgroup);
+                }
+
                 return lock;
             }
         }
@@ -182,50 +231,16 @@ struct CgroupUserLock : UserLock
         return nullptr;
     }
 
-    std::optional<Path> cgroup;
-
-    std::optional<Path> getCgroup() override
-    {
-        if (!cgroup) {
-            /* Create a systemd cgroup since that's the minimum
-               required by systemd-nspawn. */
-            auto ourCgroups = getCgroups("/proc/self/cgroup");
-            auto systemdCgroup = ourCgroups["systemd"];
-            if (systemdCgroup == "")
-                throw Error("'systemd' cgroup does not exist");
-
-            auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup);
-
-            if (!pathExists(hostCgroup))
-                throw Error("expected cgroup directory '%s'", hostCgroup);
-
-            cgroup = fmt("%s/nix-%d", hostCgroup, uid);
-
-            destroyCgroup(*cgroup);
-
-            if (mkdir(cgroup->c_str(), 0755) == -1)
-                throw SysError("creating cgroup '%s'", *cgroup);
-
-            /* Record the cgroup in the lock file. This ensures that
-               if we subsequently get executed under a different parent
-               cgroup, we kill the previous cgroup first. */
-            if (ftruncate(fdUserLock.get(), 0) == -1)
-                throw Error("truncating user lock");
-            writeFull(fdUserLock.get(), *cgroup);
-        }
-
-        return cgroup;
-    };
-};
-#endif
-
-std::unique_ptr<UserLock> acquireUserLock()
-{
     #if __linux__
-    if (settings.autoAllocateUids)
-        return CgroupUserLock::acquire();
-    else
+    std::optional<Path> getCgroup() override { return cgroup; }
     #endif
+};
+
+std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds)
+{
+    if (settings.autoAllocateUids)
+        return AutoUserLock::acquire(nrIds);
+    else
         return SimpleUserLock::acquire();
 }
 
diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh
index 4b6d34069..62676a523 100644
--- a/src/libstore/lock.hh
+++ b/src/libstore/lock.hh
@@ -11,18 +11,16 @@ struct UserLock
     virtual ~UserLock() { }
 
     /* Get the first and last UID. */
-    virtual std::pair<uid_t, uid_t> getUIDRange() = 0;
+    std::pair<uid_t, uid_t> getUIDRange()
+    {
+        auto first = getUID();
+        return {first, first + getUIDCount() - 1};
+    }
 
     /* Get the first UID. */
-    uid_t getUID()
-    {
-        return getUIDRange().first;
-    }
+    virtual uid_t getUID() = 0;
 
-    uid_t getUIDCount()
-    {
-        return getUIDRange().second - getUIDRange().first + 1;
-    }
+    virtual uid_t getUIDCount() = 0;
 
     virtual gid_t getGID() = 0;
 
@@ -31,12 +29,14 @@ struct UserLock
     /* Kill any processes currently executing as this user. */
     virtual void kill() = 0;
 
+    #if __linux__
     virtual std::optional<Path> getCgroup() { return {}; };
+    #endif
 };
 
-/* Acquire a user lock. Note that this may return nullptr if no user
-   is available. */
-std::unique_ptr<UserLock> acquireUserLock();
+/* Acquire a user lock for a UID range of size `nrIds`. Note that this
+   may return nullptr if no user is available. */
+std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds);
 
 bool useBuildUsers();
 
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index f2288a04e..487dbcfbb 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -90,6 +90,7 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
 
 StringSet ParsedDerivation::getRequiredSystemFeatures() const
 {
+    // FIXME: cache this?
     StringSet res;
     for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
         res.insert(i);
@@ -125,6 +126,11 @@ bool ParsedDerivation::substitutesAllowed() const
     return getBoolAttr("allowSubstitutes", true);
 }
 
+bool ParsedDerivation::useUidRange() const
+{
+    return getRequiredSystemFeatures().count("uid-range");
+}
+
 static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index 95bec21e8..bfb3857c0 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -38,6 +38,8 @@ public:
 
     bool substitutesAllowed() const;
 
+    bool useUidRange() const;
+
     std::optional<nlohmann::json> prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths);
 };
 

From 05d258667d12b2decda87024a59250c43343b509 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 8 Nov 2022 08:00:29 -0800
Subject: [PATCH 062/112] Fix build on macOS

---
 src/libstore/build/local-derivation-goal.cc |  2 +-
 src/libstore/globals.hh                     | 26 +++++++++++++++++----
 src/libstore/lock.cc                        | 10 ++++----
 src/libstore/lock.hh                        |  2 ++
 4 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 09da87476..45ea9968f 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -659,7 +659,7 @@ void LocalDerivationGoal::startBuilder()
         }
 
 #else
-        if (useUidRange)
+        if (parsedDrv->useUidRange())
             throw Error("feature 'uid-range' is not supported on this platform");
         if (useSystemdCgroup)
             throw Error("feature 'systemd-cgroup' is not supported on this platform");
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index be741a830..88fe72202 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -46,7 +46,13 @@ struct PluginFilesSetting : public BaseSetting<Paths>
     void set(const std::string & str, bool append = false) override;
 };
 
-const uint32_t maxIdsPerBuild = 1 << 16;
+const uint32_t maxIdsPerBuild =
+    #if __linux__
+    1 << 16
+    #else
+    1
+    #endif
+    ;
 
 class Settings : public Config {
 
@@ -277,16 +283,26 @@ public:
           multi-user settings with untrusted users.
         )"};
 
-    #if __linux__
     Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids",
         "Whether to allocate UIDs for builders automatically."};
 
-    Setting<uint32_t> startId{this, 872415232, "start-id",
+    Setting<uint32_t> startId{this,
+        #if __linux__
+        872415232,
+        #else
+        56930,
+        #endif
+        "start-id",
         "The first UID and GID to use for dynamic ID allocation."};
 
-    Setting<uint32_t> uidCount{this, maxIdsPerBuild * 128, "id-count",
+    Setting<uint32_t> uidCount{this,
+        #if __linux__
+        maxIdsPerBuild * 128,
+        #else
+        128,
+        #endif
+        "id-count",
         "The number of UIDs/GIDs to use for dynamic ID allocation."};
-    #endif
 
     Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
         "Whether to impersonate a Linux 2.6 machine on newer kernels.",
diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index ecc51cebe..f9892bb91 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -122,15 +122,16 @@ struct AutoUserLock : UserLock
 
     ~AutoUserLock()
     {
+        #if __linux__
         // Get rid of our cgroup, ignoring errors.
         if (cgroup) rmdir(cgroup->c_str());
+        #endif
     }
 
     void kill() override
     {
         #if __linux__
         if (cgroup) {
-            printError("KILL CGROUP %s", *cgroup);
             destroyCgroup(*cgroup);
             if (mkdir(cgroup->c_str(), 0755) == -1)
                 throw SysError("creating cgroup '%s'", *cgroup);
@@ -138,7 +139,6 @@ struct AutoUserLock : UserLock
         #endif
         {
             assert(firstUid);
-            printError("KILL USER %d", firstUid);
             killUser(firstUid);
         }
     }
@@ -201,6 +201,7 @@ struct AutoUserLock : UserLock
                 lock->firstUid = settings.startId + i * maxIdsPerBuild;
                 lock->nrIds = nrIds;
 
+                #if __linux__
                 if (nrIds > 1) {
                     auto ourCgroups = getCgroups("/proc/self/cgroup");
                     auto ourCgroup = ourCgroups[""];
@@ -209,20 +210,17 @@ struct AutoUserLock : UserLock
 
                     auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup);
 
-                    printError("PARENT CGROUP = %s", ourCgroupPath);
-
                     if (!pathExists(ourCgroupPath))
                         throw Error("expected cgroup directory '%s'", ourCgroupPath);
 
                     lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid);
 
-                    printError("CHILD CGROUP = %s", *lock->cgroup);
-
                     /* Record the cgroup in the lock file. This ensures that
                        if we subsequently get executed under a different parent
                        cgroup, we kill the previous cgroup first. */
                     writeFull(lock->fdUserLock.get(), *lock->cgroup);
                 }
+                #endif
 
                 return lock;
             }
diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh
index 62676a523..b5536408c 100644
--- a/src/libstore/lock.hh
+++ b/src/libstore/lock.hh
@@ -4,6 +4,8 @@
 
 #include <optional>
 
+#include <sys/types.h>
+
 namespace nix {
 
 struct UserLock

From daedaa197dbbb688323f2b22b898304c24f9c051 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 9 Nov 2022 00:49:34 +0100
Subject: [PATCH 063/112] fix typos in comments

---
 doc/manual/redirects.js | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index d9b27d866..2e77edd0f 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -2,12 +2,12 @@
 // this must be done on the client side, as web servers do not see the anchor part of the URL.
 
 // redirections are declared as follows:
-// each entry has as key the matched URL path relative to the mdBook document root.
+// each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
 //
 //     IMPORTANT: it must specify the full path with file name and suffix
 //
-// each entry is a set of key-value pairs, where
-// - keys are anchors on to the matched path.
+// each entry is itself a set of key-value pairs, where
+// - keys are anchors on the matched path.
 // - values are redirection targets relative to the current path.
 
 const redirects = {
@@ -352,16 +352,16 @@ const redirects = {
 //
 // matching URLs is more involved than it should be:
 //
-// 1. `document.location.pathname` can have an have an arbitrary prefix.
+// 1. `document.location.pathname` can have an arbitrary prefix.
 //
-// 2. `path_to_root` is set by mdBook and consists only of `../`s and
+// 2. `path_to_root` is set by mdBook. it consists only of `../`s and
 //    determines the depth of `<path>` relative to the prefix:
 //
 //          `document.location.pathname`
 //        |------------------------------|
 //        /<prefix>/<path>/[<file>[.html]][#<anchor>]
 //                  |----|
-//              `path_to_root` has same number of segments
+//              `path_to_root` has same number of path segments
 //
 //    source: https://phaiax.github.io/mdBook/format/theme/index-hbs.html#data
 //
@@ -410,7 +410,9 @@ const path = segments.join('/');
 
 // anchor starts with the hash character (`#`),
 // but our redirect declarations don't, so we strip it.
-// example: document.location.hash -> '#foo'
+// example:
+//     document.location.hash -> '#foo'
+//     document.location.hash.substring(1) -> 'foo'
 const anchor = document.location.hash.substring(1);
 
 const redirect = redirects[path];

From d8781c4fc5143153812e056df23183bcab505253 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 9 Nov 2022 01:11:47 +0100
Subject: [PATCH 064/112] add removing users to uninstall instructions

---
 .../src/installation/installing-binary.md     | 29 +++++++++++++++----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
index 2d007ca1b..615d862a4 100644
--- a/doc/manual/src/installation/installing-binary.md
+++ b/doc/manual/src/installation/installing-binary.md
@@ -88,10 +88,24 @@ extension. The installer will also create `/etc/profile.d/nix.sh`.
 
 ### Linux
 
-```console
-sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+Remove files created by Nix:
 
-# If you are on Linux with systemd, you will need to run:
+```console
+sudo rm -rf /nix /etc/nix /etc/profile/nix.sh ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+```
+
+Remove build users and their group:
+
+```console
+for i in $(seq 30001 30032); do
+  sudo userdel $i
+done
+sudo groupdel 30000
+```
+
+If you are on Linux with systemd, remove the Nix daemon service:
+
+```console
 sudo systemctl stop nix-daemon.socket
 sudo systemctl stop nix-daemon.service
 sudo systemctl disable nix-daemon.socket
@@ -99,8 +113,13 @@ sudo systemctl disable nix-daemon.service
 sudo systemctl daemon-reload
 ```
 
-There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
-and `/etc/zshrc` which you may remove.
+There may also be references to Nix in
+
+- `/etc/profile`
+- `/etc/bashrc`,
+- `/etc/zshrc`
+
+which you may remove.
 
 ### macOS
 

From 6c6eff8ac40e2f5d7b6ff8e772feebb1aa484039 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Nov 2022 17:24:12 +0100
Subject: [PATCH 065/112] Remove the SystemdCgroup feature

---
 src/libstore/build/local-derivation-goal.cc | 23 +++++++--------------
 src/libstore/build/local-derivation-goal.hh |  4 ----
 src/libutil/experimental-features.cc        |  1 -
 src/libutil/experimental-features.hh        |  1 -
 4 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 45ea9968f..e652c425c 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -495,9 +495,6 @@ void LocalDerivationGoal::startBuilder()
         }
     }
 
-    useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("Systemd-cgroup");
-    assert(!useSystemdCgroup);
-
     if (useChroot) {
 
         /* Allow a user-configurable set of directories from the
@@ -649,20 +646,18 @@ void LocalDerivationGoal::startBuilder()
                 dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
         }
 
-        if (useSystemdCgroup) {
-            settings.requireExperimentalFeature(Xp::SystemdCgroup);
-            std::optional<Path> cgroup;
-            if (!buildUser || !(cgroup = buildUser->getCgroup()))
-                throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf");
-            chownToBuilder(*cgroup);
-            chownToBuilder(*cgroup + "/cgroup.procs");
+        if (buildUser) {
+            if (auto cgroup = buildUser->getCgroup()) {
+                chownToBuilder(*cgroup);
+                chownToBuilder(*cgroup + "/cgroup.procs");
+                chownToBuilder(*cgroup + "/cgroup.threads");
+                //chownToBuilder(*cgroup + "/cgroup.subtree_control");
+            }
         }
 
 #else
         if (parsedDrv->useUidRange())
             throw Error("feature 'uid-range' is not supported on this platform");
-        if (useSystemdCgroup)
-            throw Error("feature 'systemd-cgroup' is not supported on this platform");
         #if __APPLE__
             /* We don't really have any parent prep work to do (yet?)
                All work happens in the child, instead. */
@@ -673,8 +668,6 @@ void LocalDerivationGoal::startBuilder()
     } else {
         if (parsedDrv->useUidRange())
             throw Error("feature 'uid-range' is only supported in sandboxed builds");
-        if (useSystemdCgroup)
-            throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds");
     }
 
     if (needsHashRewrite() && pathExists(homeDir))
@@ -1845,7 +1838,7 @@ void LocalDerivationGoal::runChild()
             /* Unshare the cgroup namespace. This means
                /proc/self/cgroup will show the child's cgroup as '/'
                rather than whatever it is in the parent. */
-            if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1)
+            if (buildUser && buildUser->getUIDCount() != 1 && unshare(CLONE_NEWCGROUP) == -1)
                 throw SysError("unsharing cgroup namespace");
 
             /* Do the chroot(). */
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 61b0f9145..070ae53f3 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -41,10 +41,6 @@ struct LocalDerivationGoal : public DerivationGoal
 
     Path chrootRootDir;
 
-    /* Whether to make the 'systemd' cgroup controller available to
-       the build. */
-    bool useSystemdCgroup = false;
-
     /* RAII object to delete the chroot directory. */
     std::shared_ptr<AutoDelete> autoDelChroot;
 
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 670079019..0f05f3752 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -15,7 +15,6 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::FetchClosure, "fetch-closure" },
     { Xp::ReplFlake, "repl-flake" },
     { Xp::AutoAllocateUids, "auto-allocate-uids" },
-    { Xp::SystemdCgroup, "systemd-cgroup" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index c749d4767..cf0c06eac 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -24,7 +24,6 @@ enum struct ExperimentalFeature
     FetchClosure,
     ReplFlake,
     AutoAllocateUids,
-    SystemdCgroup,
 };
 
 /**

From e7ed9ae0c711c4efd83756b16379549ecff52355 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Na=C3=AFm=20Favier?= <n@monade.li>
Date: Fri, 4 Nov 2022 14:19:31 +0100
Subject: [PATCH 066/112] Restrict `readFile` context to references that appear
 in the string

When calling `builtins.readFile` on a store path, the references of that
path are currently added to the resulting string's context.

This change makes those references the *possible* context of the string,
but filters them to keep only the references whose hash actually appears
in the string, similarly to what is done for determining the runtime
references of a path.
---
 src/libexpr/primops.cc     |  5 ++++
 src/libstore/references.cc | 57 +++++++++++++++++++++++++-------------
 src/libstore/references.hh | 13 +++++++++
 3 files changed, 55 insertions(+), 20 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 28b998474..ff620ca63 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -5,6 +5,7 @@
 #include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"
+#include "references.hh"
 #include "store-api.hh"
 #include "util.hh"
 #include "json.hh"
@@ -1542,6 +1543,10 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V
             refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references;
         } catch (Error &) { // FIXME: should be InvalidPathError
         }
+        // Re-scan references to filter down to just the ones that actually occur in the file.
+        auto refsSink = PathRefScanSink::fromPaths(refs);
+        refsSink << s;
+        refs = refsSink.getResultPaths();
     }
     auto context = state.store->printStorePathSet(refs);
     v.mkString(s, context);
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 34dce092c..3bb297fc8 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -67,6 +67,40 @@ void RefScanSink::operator () (std::string_view data)
 }
 
 
+PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
+    : RefScanSink(std::move(hashes))
+    , backMap(std::move(backMap))
+{ }
+
+PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
+{
+    StringSet hashes;
+    std::map<std::string, StorePath> backMap;
+
+    for (auto & i : refs) {
+        std::string hashPart(i.hashPart());
+        auto inserted = backMap.emplace(hashPart, i).second;
+        assert(inserted);
+        hashes.insert(hashPart);
+    }
+
+    return PathRefScanSink(std::move(hashes), std::move(backMap));
+}
+
+StorePathSet PathRefScanSink::getResultPaths()
+{
+    /* Map the hashes found back to their store paths. */
+    StorePathSet found;
+    for (auto & i : getResult()) {
+        auto j = backMap.find(i);
+        assert(j != backMap.end());
+        found.insert(j->second);
+    }
+
+    return found;
+}
+
+
 std::pair<StorePathSet, HashResult> scanForReferences(
     const std::string & path,
     const StorePathSet & refs)
@@ -82,30 +116,13 @@ StorePathSet scanForReferences(
     const Path & path,
     const StorePathSet & refs)
 {
-    StringSet hashes;
-    std::map<std::string, StorePath> backMap;
-
-    for (auto & i : refs) {
-        std::string hashPart(i.hashPart());
-        auto inserted = backMap.emplace(hashPart, i).second;
-        assert(inserted);
-        hashes.insert(hashPart);
-    }
+    PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
+    TeeSink sink { refsSink, toTee };
 
     /* Look for the hashes in the NAR dump of the path. */
-    RefScanSink refsSink(std::move(hashes));
-    TeeSink sink { refsSink, toTee };
     dumpPath(path, sink);
 
-    /* Map the hashes found back to their store paths. */
-    StorePathSet found;
-    for (auto & i : refsSink.getResult()) {
-        auto j = backMap.find(i);
-        assert(j != backMap.end());
-        found.insert(j->second);
-    }
-
-    return found;
+    return refsSink.getResultPaths();
 }
 
 
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index a6119c861..6f381f96c 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -27,6 +27,19 @@ public:
     void operator () (std::string_view data) override;
 };
 
+class PathRefScanSink : public RefScanSink
+{
+    std::map<std::string, StorePath> backMap;
+
+    PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
+
+public:
+
+    static PathRefScanSink fromPaths(const StorePathSet & refs);
+
+    StorePathSet getResultPaths();
+};
+
 struct RewritingSink : Sink
 {
     std::string from, to, prev;

From 2af036e5a378c711d8e58d01bdefe5c634c25921 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 11 Nov 2022 14:01:13 +0100
Subject: [PATCH 067/112] remove stray comma

---
 doc/manual/src/installation/installing-binary.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
index 615d862a4..31faeadc2 100644
--- a/doc/manual/src/installation/installing-binary.md
+++ b/doc/manual/src/installation/installing-binary.md
@@ -116,7 +116,7 @@ sudo systemctl daemon-reload
 There may also be references to Nix in
 
 - `/etc/profile`
-- `/etc/bashrc`,
+- `/etc/bashrc`
 - `/etc/zshrc`
 
 which you may remove.

From 07f2cb1e8f03784041475c27c2ba0aac7be6c0b7 Mon Sep 17 00:00:00 2001
From: Tobias Mayer <tobim@fastmail.fm>
Date: Thu, 10 Nov 2022 08:59:23 +0100
Subject: [PATCH 068/112] libstore: link to aws-crt-cpp

This change is needed to support aws-sdk-cpp 1.10 and newer.

I opted not to make this dependent on the sdk version because
the crt dependency has been in the interface of the older
sdk as well, and it was only coincidence that libstore didn't
make use of any privately defined symbols directly.
---
 src/libstore/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 1d26ac918..8f28bec6c 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -20,7 +20,7 @@ endif
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 
 ifeq ($(ENABLE_S3), 1)
-	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
+	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp
 endif
 
 ifdef HOST_SOLARIS

From efadeee8fd593cd9457c75299165d5d5ac159d0f Mon Sep 17 00:00:00 2001
From: Et7f3 <cadeaudeelie@gmail.com>
Date: Sat, 12 Nov 2022 23:04:58 +0100
Subject: [PATCH 069/112] build: use pkg-config for lowdown

---
 src/libcmd/local.mk | 2 +-
 src/nix/local.mk    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
index 3a4de6bcb..152bc388d 100644
--- a/src/libcmd/local.mk
+++ b/src/libcmd/local.mk
@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix
 
-libcmd_LDFLAGS = $(EDITLINE_LIBS) -llowdown -pthread
+libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
diff --git a/src/nix/local.mk b/src/nix/local.mk
index e4ec7634d..0f2f016ec 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -18,7 +18,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \

From 6bf873651740b1552dea6f30c7778dff11bc52ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 14 Nov 2022 15:03:53 +0100
Subject: [PATCH 070/112] Add release-notes for the context-restriction in
 readFile

---
 doc/manual/src/release-notes/rl-next.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 68f7d1a9d..2069e4578 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -5,3 +5,8 @@
   arguments will be ignored and the resulting derivation will have
   `__impure` set to `true`, making it an impure derivation.
 
+* If `builtins.readFile` is called on a file with context, then only the parts
+  of that context that appear in the content of the file are retained.
+  This avoids a lot of spurious errors where some benign strings end-up having
+  a context just because they are read from a store path
+  ([#7260](https://github.com/NixOS/nix/pull/7260)).

From cb39e9a99e21812b424d7d2318157163ab97fc82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 14 Nov 2022 15:13:46 +0100
Subject: [PATCH 071/112] Test that the result of `readFile` gets ref-scanned

---
 tests/readfile-context.builder.sh |  1 -
 tests/readfile-context.nix        | 19 ++++++++++++++-----
 2 files changed, 14 insertions(+), 6 deletions(-)
 delete mode 100644 tests/readfile-context.builder.sh

diff --git a/tests/readfile-context.builder.sh b/tests/readfile-context.builder.sh
deleted file mode 100644
index 7084a08cb..000000000
--- a/tests/readfile-context.builder.sh
+++ /dev/null
@@ -1 +0,0 @@
-echo "$input" > $out
diff --git a/tests/readfile-context.nix b/tests/readfile-context.nix
index 600036a94..54cd1afd9 100644
--- a/tests/readfile-context.nix
+++ b/tests/readfile-context.nix
@@ -6,14 +6,23 @@ let
 
   dependent = mkDerivation {
     name = "dependent";
-    builder = ./readfile-context.builder.sh;
-    input = "${input}/hello";
+    buildCommand = ''
+      mkdir -p $out
+      echo -n "$input1" > "$out/file1"
+      echo -n "$input2" > "$out/file2"
+    '';
+    input1 = "${input}/hello";
+    input2 = "hello";
   };
 
   readDependent = mkDerivation {
-    name = "read-dependent";
-    builder = ./readfile-context.builder.sh;
-    input = builtins.readFile dependent;
+    # Will evaluate correctly because file2 doesn't have any references,
+    # even though the `dependent` derivation does.
+    name = builtins.readFile (dependent + "/file2");
+    buildCommand = ''
+      echo "$input" > "$out"
+    '';
+    input = builtins.readFile (dependent + "/file1");
   };
 
 in readDependent

From 7e162c69fe6cbfb929b5356a7df9de5c25c22565 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 14 Nov 2022 16:26:20 +0100
Subject: [PATCH 072/112] derivation-goal: Fix `requires non-existing output`
 error

It occurred when a output of the dependency was already available,
so it didn't need rebuilding and didn't get added to the
inputDrvOutputs.
This process-related info wasn't suitable for the purpose of finding
the actual input paths for the builder. It is better to do this in
absolute terms by querying the store.
---
 src/libstore/build/derivation-goal.cc | 30 +++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 41d2e2a1c..00e375fe9 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -528,13 +528,31 @@ void DerivationGoal::inputsRealised()
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            for (auto & j : wantedDepOutputs)
-                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
+            for (auto & j : wantedDepOutputs) {
+                /* TODO (impure derivations-induced tech debt):
+                   Tracking input derivation outputs statefully through the
+                   goals is error prone and has led to bugs.
+                   For a robust nix, we need to move towards the `else` branch,
+                   which does not rely on goal state to match up with the
+                   reality of the store, which is our real source of truth.
+                   However, the impure derivations feature still relies on this
+                   fragile way of doing things, because its builds do not have
+                   a representation in the store, which is a usability problem
+                   in itself */
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
                     worker.store.computeFSClosure(*outPath, inputPaths);
-                else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                }
+                else {
+                    auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
+                    auto outMapPath = outMap.find(j);
+                    if (outMapPath == outMap.end()) {
+                        throw Error(
+                            "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
+                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                    }
+                    worker.store.computeFSClosure(outMapPath->second, inputPaths);
+                }
+            }
         }
     }
 

From c279ddb18cf3a34b0f6d4e3adcf9455da5397ad7 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 9 Nov 2022 15:21:13 +0100
Subject: [PATCH 073/112] tests: Reproduce #6572

---
 tests/build.sh             | 51 ++++++++++++++++++++++++++++++++++++++
 tests/multiple-outputs.nix | 30 ++++++++++++++++++++++
 2 files changed, 81 insertions(+)

diff --git a/tests/build.sh b/tests/build.sh
index fc6825e25..c7db039b4 100644
--- a/tests/build.sh
+++ b/tests/build.sh
@@ -70,3 +70,54 @@ testNormalization () {
 }
 
 testNormalization
+
+# https://github.com/NixOS/nix/issues/6572
+issue_6572_independent_outputs() {
+    nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json
+
+    # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation.
+    p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
+    nix-store --delete "$p" # Clean up for next test
+
+    # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
+    nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)"
+    p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
+    cmp $p <<EOF
+first
+second
+EOF
+    nix-store --delete "$p" # Clean up for next test
+
+    # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
+    nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.second)"
+    p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
+    cmp $p <<EOF
+first
+second
+EOF
+    nix-store --delete "$p" # Clean up for next test
+}
+issue_6572_independent_outputs
+
+
+# https://github.com/NixOS/nix/issues/6572
+issue_6572_dependent_outputs() {
+
+    nix build -f multiple-outputs.nix --json a --no-link > $TEST_ROOT/a.json
+
+    # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation.
+    p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths)
+    nix-store --delete "$p" # Clean up for next test
+
+    # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
+    nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)"
+    p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths)
+    cmp $p <<EOF
+first
+second
+EOF
+    nix-store --delete "$p" # Clean up for next test
+}
+if isDaemonNewer "2.12pre0"; then
+    issue_6572_dependent_outputs
+fi
diff --git a/tests/multiple-outputs.nix b/tests/multiple-outputs.nix
index 624a5dade..1429bc648 100644
--- a/tests/multiple-outputs.nix
+++ b/tests/multiple-outputs.nix
@@ -31,6 +31,15 @@ rec {
     helloString = "Hello, world!";
   };
 
+  use-a = mkDerivation {
+    name = "use-a";
+    inherit (a) first second;
+    builder = builtins.toFile "builder.sh"
+      ''
+        cat $first/file $second/file >$out
+      '';
+  };
+
   b = mkDerivation {
     defaultOutput = assert a.second.helloString == "Hello, world!"; a;
     firstOutput = assert a.outputName == "first"; a.first.first;
@@ -87,4 +96,25 @@ rec {
     buildCommand = "mkdir $a $b $c";
   };
 
+  independent = mkDerivation {
+    name = "multiple-outputs-independent";
+    outputs = [ "first" "second" ];
+    builder = builtins.toFile "builder.sh"
+      ''
+        mkdir $first $second
+        test -z $all
+        echo "first" > $first/file
+        echo "second" > $second/file
+      '';
+  };
+
+  use-independent = mkDerivation {
+    name = "use-independent";
+    inherit (a) first second;
+    builder = builtins.toFile "builder.sh"
+      ''
+        cat $first/file $second/file >$out
+      '';
+  };
+
 }

From 60dea270d0bc430930f5560ebac71a2dc0ab2b21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Wed, 16 Nov 2022 10:34:32 +0100
Subject: [PATCH 074/112] Swallow the error in a more idiomatic way

---
 src/libcmd/installables.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index e8836c247..f63b9eeae 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -258,9 +258,8 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
                 getDefaultFlakeAttrPaths(),
                 prefix);
         }
-    } catch (EvalError& e) {
-        // swallow eval error
-        (void)e;
+    } catch (EvalError&) {
+        // Don't want eval errors to mess-up with the completion engine, so let's just swallow them
     }
 }
 

From 09f00dd4d01aa1b6866978d162022133e521614f Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Wed, 16 Nov 2022 16:49:49 +0100
Subject: [PATCH 075/112] Replace src/libutil/json.cc with nlohmann json
 generation

---
 src/libexpr/eval.cc                         | 144 +++++++-------
 src/libexpr/primops.cc                      |  13 +-
 src/libexpr/value-to-json.cc                |  50 +++--
 src/libexpr/value-to-json.hh                |   7 +-
 src/libexpr/value.hh                        |   6 +-
 src/libstore/binary-cache-store.cc          |  18 +-
 src/libstore/build/derivation-goal.cc       |   1 -
 src/libstore/build/local-derivation-goal.cc |   2 +-
 src/libstore/nar-accessor.cc                |  30 +--
 src/libstore/nar-accessor.hh                |   6 +-
 src/libstore/parsed-derivations.cc          |  16 +-
 src/libstore/remote-fs-accessor.cc          |   8 +-
 src/libstore/store-api.cc                   |  48 ++---
 src/libstore/store-api.hh                   |   4 +-
 src/libutil/json.cc                         | 203 --------------------
 src/libutil/json.hh                         | 185 ------------------
 src/libutil/tests/json.cc                   | 193 -------------------
 src/nix-env/nix-env.cc                      |  35 ++--
 src/nix/eval.cc                             |   7 +-
 src/nix/flake.cc                            |  37 ++--
 src/nix/ls.cc                               |   5 +-
 src/nix/make-content-addressed.cc           |  12 +-
 src/nix/path-info.cc                        |   8 +-
 src/nix/search.cc                           |  19 +-
 src/nix/show-derivation.cc                  |  67 +++----
 25 files changed, 266 insertions(+), 858 deletions(-)
 delete mode 100644 src/libutil/json.cc
 delete mode 100644 src/libutil/json.hh
 delete mode 100644 src/libutil/tests/json.cc

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 563f24e48..e78d28b97 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -7,7 +7,6 @@
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "filetransfer.hh"
-#include "json.hh"
 #include "function-trace.hh"
 
 #include <algorithm>
@@ -21,6 +20,7 @@
 #include <functional>
 
 #include <sys/resource.h>
+#include <nlohmann/json.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -35,6 +35,8 @@
 
 #endif
 
+using json = nlohmann::json;
+
 namespace nix {
 
 static char * allocString(size_t size)
@@ -2441,97 +2443,97 @@ void EvalState::printStats()
         std::fstream fs;
         if (outPath != "-")
             fs.open(outPath, std::fstream::out);
-        JSONObject topObj(outPath == "-" ? std::cerr : fs, true);
-        topObj.attr("cpuTime",cpuTime);
-        {
-            auto envs = topObj.object("envs");
-            envs.attr("number", nrEnvs);
-            envs.attr("elements", nrValuesInEnvs);
-            envs.attr("bytes", bEnvs);
-        }
-        {
-            auto lists = topObj.object("list");
-            lists.attr("elements", nrListElems);
-            lists.attr("bytes", bLists);
-            lists.attr("concats", nrListConcats);
-        }
-        {
-            auto values = topObj.object("values");
-            values.attr("number", nrValues);
-            values.attr("bytes", bValues);
-        }
-        {
-            auto syms = topObj.object("symbols");
-            syms.attr("number", symbols.size());
-            syms.attr("bytes", symbols.totalSize());
-        }
-        {
-            auto sets = topObj.object("sets");
-            sets.attr("number", nrAttrsets);
-            sets.attr("bytes", bAttrsets);
-            sets.attr("elements", nrAttrsInAttrsets);
-        }
-        {
-            auto sizes = topObj.object("sizes");
-            sizes.attr("Env", sizeof(Env));
-            sizes.attr("Value", sizeof(Value));
-            sizes.attr("Bindings", sizeof(Bindings));
-            sizes.attr("Attr", sizeof(Attr));
-        }
-        topObj.attr("nrOpUpdates", nrOpUpdates);
-        topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied);
-        topObj.attr("nrThunks", nrThunks);
-        topObj.attr("nrAvoided", nrAvoided);
-        topObj.attr("nrLookups", nrLookups);
-        topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
-        topObj.attr("nrFunctionCalls", nrFunctionCalls);
+        json topObj = json::object();
+        topObj["cpuTime"] = cpuTime;
+        topObj["envs"] = {
+            {"number", nrEnvs},
+            {"elements", nrValuesInEnvs},
+            {"bytes", bEnvs},
+        };
+        topObj["list"] = {
+            {"elements", nrListElems},
+            {"bytes", bLists},
+            {"concats", nrListConcats},
+        };
+        topObj["values"] = {
+            {"number", nrValues},
+            {"bytes", bValues},
+        };
+        topObj["symbols"] = {
+            {"number", symbols.size()},
+            {"bytes", symbols.totalSize()},
+        };
+        topObj["sets"] = {
+            {"number", nrAttrsets},
+            {"bytes", bAttrsets},
+            {"elements", nrAttrsInAttrsets},
+        };
+        topObj["sizes"] = {
+            {"Env", sizeof(Env)},
+            {"Value", sizeof(Value)},
+            {"Bindings", sizeof(Bindings)},
+            {"Attr", sizeof(Attr)},
+        };
+        topObj["nrOpUpdates"] = nrOpUpdates;
+        topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
+        topObj["nrThunks"] = nrThunks;
+        topObj["nrAvoided"] = nrAvoided;
+        topObj["nrLookups"] = nrLookups;
+        topObj["nrPrimOpCalls"] = nrPrimOpCalls;
+        topObj["nrFunctionCalls"] = nrFunctionCalls;
 #if HAVE_BOEHMGC
-        {
-            auto gc = topObj.object("gc");
-            gc.attr("heapSize", heapSize);
-            gc.attr("totalBytes", totalBytes);
-        }
+        topObj["gc"] = {
+            {"heapSize", heapSize},
+            {"totalBytes", totalBytes},
+        };
 #endif
 
         if (countCalls) {
+            topObj["primops"] = primOpCalls;
             {
-                auto obj = topObj.object("primops");
-                for (auto & i : primOpCalls)
-                    obj.attr(i.first, i.second);
-            }
-            {
-                auto list = topObj.list("functions");
+                auto& list = topObj["functions"];
+                list = json::array();
                 for (auto & [fun, count] : functionCalls) {
-                    auto obj = list.object();
+                    json obj = json::object();
                     if (fun->name)
-                        obj.attr("name", (std::string_view) symbols[fun->name]);
+                        obj["name"] = (std::string_view) symbols[fun->name];
                     else
-                        obj.attr("name", nullptr);
+                        obj["name"] = nullptr;
                     if (auto pos = positions[fun->pos]) {
-                        obj.attr("file", (std::string_view) pos.file);
-                        obj.attr("line", pos.line);
-                        obj.attr("column", pos.column);
+                        obj["file"] = (std::string_view) pos.file;
+                        obj["line"] = pos.line;
+                        obj["column"] = pos.column;
                     }
-                    obj.attr("count", count);
+                    obj["count"] = count;
+                    list.push_back(obj);
                 }
             }
             {
-                auto list = topObj.list("attributes");
+                auto list = topObj["attributes"];
+                list = json::array();
                 for (auto & i : attrSelects) {
-                    auto obj = list.object();
+                    json obj = json::object();
                     if (auto pos = positions[i.first]) {
-                        obj.attr("file", (const std::string &) pos.file);
-                        obj.attr("line", pos.line);
-                        obj.attr("column", pos.column);
+                        obj["file"] = (const std::string &) pos.file;
+                        obj["line"] = pos.line;
+                        obj["column"] = pos.column;
                     }
-                    obj.attr("count", i.second);
+                    obj["count"] = i.second;
+                    list.push_back(obj);
                 }
             }
         }
 
         if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") {
-            auto list = topObj.list("symbols");
-            symbols.dump([&](const std::string & s) { list.elem(s); });
+            // XXX: overrides earlier assignment
+            topObj["symbols"] = json::array();
+            auto &list = topObj["symbols"];
+            symbols.dump([&](const std::string & s) { list.emplace_back(s); });
+        }
+        if (outPath == "-") {
+            std::cerr << topObj.dump(2) << std::endl;
+        } else {
+            fs << topObj.dump(2) << std::endl;
         }
     }
 }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 22f6ad3cc..05265411c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -8,12 +8,12 @@
 #include "references.hh"
 #include "store-api.hh"
 #include "util.hh"
-#include "json.hh"
 #include "value-to-json.hh"
 #include "value-to-xml.hh"
 #include "primops.hh"
 
 #include <boost/container/small_vector.hpp>
+#include <nlohmann/json.hpp>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -1011,6 +1011,7 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val
    derivation. */
 static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
+    using nlohmann::json;
     state.forceAttrs(*args[0], pos);
 
     /* Figure out the name first (for stack backtraces). */
@@ -1032,11 +1033,10 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
     }
 
     /* Check whether attributes should be passed as a JSON file. */
-    std::ostringstream jsonBuf;
-    std::unique_ptr<JSONObject> jsonObject;
+    std::optional<json> jsonObject;
     attr = args[0]->attrs->find(state.sStructuredAttrs);
     if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos))
-        jsonObject = std::make_unique<JSONObject>(jsonBuf);
+        jsonObject = json::object();
 
     /* Check whether null attributes should be ignored. */
     bool ignoreNulls = false;
@@ -1138,8 +1138,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
 
                     if (i->name == state.sStructuredAttrs) continue;
 
-                    auto placeholder(jsonObject->placeholder(key));
-                    printValueAsJSON(state, true, *i->value, pos, placeholder, context);
+                    (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context);
 
                     if (i->name == state.sBuilder)
                         drv.builder = state.forceString(*i->value, context, posDrvName);
@@ -1183,8 +1182,8 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
     }
 
     if (jsonObject) {
+        drv.env.emplace("__json", jsonObject->dump());
         jsonObject.reset();
-        drv.env.emplace("__json", jsonBuf.str());
     }
 
     /* Everything in the context of the strings in the derivation
diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc
index 4d63d8b49..5dc453b2e 100644
--- a/src/libexpr/value-to-json.cc
+++ b/src/libexpr/value-to-json.cc
@@ -1,84 +1,82 @@
 #include "value-to-json.hh"
-#include "json.hh"
 #include "eval-inline.hh"
 #include "util.hh"
 
 #include <cstdlib>
 #include <iomanip>
+#include <nlohmann/json.hpp>
 
 
 namespace nix {
-
-void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore)
+using json = nlohmann::json;
+json printValueAsJSON(EvalState & state, bool strict,
+    Value & v, const PosIdx pos, PathSet & context, bool copyToStore)
 {
     checkInterrupt();
 
     if (strict) state.forceValue(v, pos);
 
+    json out;
+
     switch (v.type()) {
 
         case nInt:
-            out.write(v.integer);
+            out = v.integer;
             break;
 
         case nBool:
-            out.write(v.boolean);
+            out = v.boolean;
             break;
 
         case nString:
             copyContext(v, context);
-            out.write(v.string.s);
+            out = v.string.s;
             break;
 
         case nPath:
             if (copyToStore)
-                out.write(state.copyPathToStore(context, v.path));
+                out = state.copyPathToStore(context, v.path);
             else
-                out.write(v.path);
+                out = v.path;
             break;
 
         case nNull:
-            out.write(nullptr);
             break;
 
         case nAttrs: {
             auto maybeString = state.tryAttrsToString(pos, v, context, false, false);
             if (maybeString) {
-                out.write(*maybeString);
+                out = *maybeString;
                 break;
             }
             auto i = v.attrs->find(state.sOutPath);
             if (i == v.attrs->end()) {
-                auto obj(out.object());
+                out = json::object();
                 StringSet names;
                 for (auto & j : *v.attrs)
                     names.emplace(state.symbols[j.name]);
                 for (auto & j : names) {
                     Attr & a(*v.attrs->find(state.symbols.create(j)));
-                    auto placeholder(obj.placeholder(j));
-                    printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore);
+                    out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore);
                 }
             } else
-                printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore);
+                return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore);
             break;
         }
 
         case nList: {
-            auto list(out.list());
-            for (auto elem : v.listItems()) {
-                auto placeholder(list.placeholder());
-                printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore);
-            }
+            out = json::array();
+            for (auto elem : v.listItems())
+                out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore));
             break;
         }
 
         case nExternal:
-            v.external->printValueAsJSON(state, strict, out, context, copyToStore);
+            return v.external->printValueAsJSON(state, strict, context, copyToStore);
             break;
 
         case nFloat:
-            out.write(v.fpoint);
+            out = v.fpoint;
             break;
 
         case nThunk:
@@ -91,17 +89,17 @@ void printValueAsJSON(EvalState & state, bool strict,
             state.debugThrowLastTrace(e);
             throw e;
     }
+    return out;
 }
 
 void printValueAsJSON(EvalState & state, bool strict,
     Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore)
 {
-    JSONPlaceholder out(str);
-    printValueAsJSON(state, strict, v, pos, out, context, copyToStore);
+    str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
 }
 
-void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
-    JSONPlaceholder & out, PathSet & context, bool copyToStore) const
+json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
+    PathSet & context, bool copyToStore) const
 {
     state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType()));
 }
diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh
index 7ddc8a5b1..22f26b790 100644
--- a/src/libexpr/value-to-json.hh
+++ b/src/libexpr/value-to-json.hh
@@ -5,13 +5,12 @@
 
 #include <string>
 #include <map>
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix {
 
-class JSONPlaceholder;
-
-void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true);
+nlohmann::json printValueAsJSON(EvalState & state, bool strict,
+    Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true);
 
 void printValueAsJSON(EvalState & state, bool strict,
     Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true);
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 590ba7783..5adac72f8 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -7,6 +7,7 @@
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
 #endif
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix {
 
@@ -62,7 +63,6 @@ class StorePath;
 class Store;
 class EvalState;
 class XMLWriter;
-class JSONPlaceholder;
 
 
 typedef int64_t NixInt;
@@ -98,8 +98,8 @@ class ExternalValueBase
     virtual bool operator ==(const ExternalValueBase & b) const;
 
     /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
-    virtual void printValueAsJSON(EvalState & state, bool strict,
-        JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const;
+    virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict,
+        PathSet & context, bool copyToStore = true) const;
 
     /* Print the value as XML. Defaults to unevaluated */
     virtual void printValueAsXML(EvalState & state, bool strict, bool location,
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index a26770c79..12d0c32fb 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -9,7 +9,6 @@
 #include "remote-fs-accessor.hh"
 #include "nar-info-disk-cache.hh"
 #include "nar-accessor.hh"
-#include "json.hh"
 #include "thread-pool.hh"
 #include "callback.hh"
 
@@ -194,19 +193,12 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
-        std::ostringstream jsonOut;
+        nlohmann::json j = {
+            {"version", 1},
+            {"root", listNar(ref<FSAccessor>(narAccessor), "", true)},
+        };
 
-        {
-            JSONObject jsonRoot(jsonOut);
-            jsonRoot.attr("version", 1);
-
-            {
-                auto res = jsonRoot.placeholder("root");
-                listNar(res, ref<FSAccessor>(narAccessor), "", true);
-            }
-        }
-
-        upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
     }
 
     /* Optionally maintain an index of DWARF debug info files
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 00e375fe9..1938f4bcb 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -7,7 +7,6 @@
 #include "finally.hh"
 #include "util.hh"
 #include "archive.hh"
-#include "json.hh"
 #include "compression.hh"
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 5cea3b590..c786e8613 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -8,7 +8,6 @@
 #include "finally.hh"
 #include "util.hh"
 #include "archive.hh"
-#include "json.hh"
 #include "compression.hh"
 #include "daemon.hh"
 #include "worker-protocol.hh"
@@ -56,6 +55,7 @@
 
 #include <pwd.h>
 #include <grp.h>
+#include <iostream>
 
 namespace nix {
 
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 398147fc3..9a0003588 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -1,6 +1,5 @@
 #include "nar-accessor.hh"
 #include "archive.hh"
-#include "json.hh"
 
 #include <map>
 #include <stack>
@@ -243,42 +242,43 @@ ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
     return make_ref<NarAccessor>(listing, getNarBytes);
 }
 
-void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
-    const Path & path, bool recurse)
+using nlohmann::json;
+json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
 {
     auto st = accessor->stat(path);
 
-    auto obj = res.object();
+    json obj = json::object();
 
     switch (st.type) {
     case FSAccessor::Type::tRegular:
-        obj.attr("type", "regular");
-        obj.attr("size", st.fileSize);
+        obj["type"] = "regular";
+        obj["size"] = st.fileSize;
         if (st.isExecutable)
-            obj.attr("executable", true);
+            obj["executable"] = true;
         if (st.narOffset)
-            obj.attr("narOffset", st.narOffset);
+            obj["narOffset"] = st.narOffset;
         break;
     case FSAccessor::Type::tDirectory:
-        obj.attr("type", "directory");
+        obj["type"] = "directory";
         {
-            auto res2 = obj.object("entries");
+            obj["entries"] = json::object();
+            json &res2 = obj["entries"];
             for (auto & name : accessor->readDirectory(path)) {
                 if (recurse) {
-                    auto res3 = res2.placeholder(name);
-                    listNar(res3, accessor, path + "/" + name, true);
+                    res2[name] = listNar(accessor, path + "/" + name, true);
                 } else
-                    res2.object(name);
+                    res2[name] = json::object();
             }
         }
         break;
     case FSAccessor::Type::tSymlink:
-        obj.attr("type", "symlink");
-        obj.attr("target", accessor->readLink(path));
+        obj["type"] = "symlink";
+        obj["target"] = accessor->readLink(path);
         break;
     default:
         throw Error("path '%s' does not exist in NAR", path);
     }
+    return obj;
 }
 
 }
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index c2241a04c..7d998ae0b 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -2,6 +2,7 @@
 
 #include <functional>
 
+#include <nlohmann/json_fwd.hpp>
 #include "fs-accessor.hh"
 
 namespace nix {
@@ -24,11 +25,8 @@ ref<FSAccessor> makeLazyNarAccessor(
     const std::string & listing,
     GetNarBytes getNarBytes);
 
-class JSONPlaceholder;
-
 /* Write a JSON representation of the contents of a NAR (except file
    contents). */
-void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
-    const Path & path, bool recurse);
+nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
 
 }
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index f2288a04e..59a30db10 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -2,7 +2,6 @@
 
 #include <nlohmann/json.hpp>
 #include <regex>
-#include "json.hh"
 
 namespace nix {
 
@@ -144,16 +143,11 @@ std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & s
     auto e = json.find("exportReferencesGraph");
     if (e != json.end() && e->is_object()) {
         for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(store.parseStorePath(p.get<std::string>()));
-                store.pathInfoToJSON(jsonRoot,
-                    store.exportReferences(storePaths, inputPaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+            StorePathSet storePaths;
+            for (auto & p : *i)
+                storePaths.insert(store.parseStorePath(p.get<std::string>()));
+            json[i.key()] = store.pathInfoToJSON(
+                store.exportReferences(storePaths, inputPaths), false, true);
         }
     }
 
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 0ce335646..fcfb527f5 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -1,6 +1,6 @@
+#include <nlohmann/json.hpp>
 #include "remote-fs-accessor.hh"
 #include "nar-accessor.hh"
-#include "json.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -38,10 +38,8 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str
 
     if (cacheDir != "") {
         try {
-            std::ostringstream str;
-            JSONPlaceholder jsonRoot(str);
-            listNar(jsonRoot, narAccessor, "", true);
-            writeFile(makeCacheFile(hashPart, "ls"), str.str());
+            nlohmann::json j = listNar(narAccessor, "", true);
+            writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreException();
         }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 06a9758fc..8811ab578 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -6,14 +6,16 @@
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
-#include "json.hh"
 #include "url.hh"
 #include "archive.hh"
 #include "callback.hh"
 #include "remote-store.hh"
 
+#include <nlohmann/json.hpp>
 #include <regex>
 
+using json = nlohmann::json;
+
 namespace nix {
 
 
@@ -838,56 +840,53 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor
     return paths;
 }
 
-
-void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
+json Store::pathInfoToJSON(const StorePathSet & storePaths,
     bool includeImpureInfo, bool showClosureSize,
     Base hashBase,
     AllowInvalidFlag allowInvalid)
 {
-    auto jsonList = jsonOut.list();
+    json::array_t jsonList = json::array();
 
     for (auto & storePath : storePaths) {
-        auto jsonPath = jsonList.object();
+        auto& jsonPath = jsonList.emplace_back(json::object());
 
         try {
             auto info = queryPathInfo(storePath);
 
-            jsonPath.attr("path", printStorePath(info->path));
-            jsonPath
-                .attr("narHash", info->narHash.to_string(hashBase, true))
-                .attr("narSize", info->narSize);
+            jsonPath["path"] = printStorePath(info->path);
+            jsonPath["narHash"] = info->narHash.to_string(hashBase, true);
+            jsonPath["narSize"] = info->narSize;
 
             {
-                auto jsonRefs = jsonPath.list("references");
+                auto& jsonRefs = (jsonPath["references"] = json::array());
                 for (auto & ref : info->references)
-                    jsonRefs.elem(printStorePath(ref));
+                    jsonRefs.emplace_back(printStorePath(ref));
             }
 
             if (info->ca)
-                jsonPath.attr("ca", renderContentAddress(info->ca));
+                jsonPath["ca"] = renderContentAddress(info->ca);
 
             std::pair<uint64_t, uint64_t> closureSizes;
 
             if (showClosureSize) {
                 closureSizes = getClosureSize(info->path);
-                jsonPath.attr("closureSize", closureSizes.first);
+                jsonPath["closureSize"] = closureSizes.first;
             }
 
             if (includeImpureInfo) {
 
                 if (info->deriver)
-                    jsonPath.attr("deriver", printStorePath(*info->deriver));
+                    jsonPath["deriver"] = printStorePath(*info->deriver);
 
                 if (info->registrationTime)
-                    jsonPath.attr("registrationTime", info->registrationTime);
+                    jsonPath["registrationTime"] = info->registrationTime;
 
                 if (info->ultimate)
-                    jsonPath.attr("ultimate", info->ultimate);
+                    jsonPath["ultimate"] = info->ultimate;
 
                 if (!info->sigs.empty()) {
-                    auto jsonSigs = jsonPath.list("signatures");
                     for (auto & sig : info->sigs)
-                        jsonSigs.elem(sig);
+                        jsonPath["signatures"].push_back(sig);
                 }
 
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
@@ -895,21 +894,22 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
 
                 if (narInfo) {
                     if (!narInfo->url.empty())
-                        jsonPath.attr("url", narInfo->url);
+                        jsonPath["url"] = narInfo->url;
                     if (narInfo->fileHash)
-                        jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
+                        jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true);
                     if (narInfo->fileSize)
-                        jsonPath.attr("downloadSize", narInfo->fileSize);
+                        jsonPath["downloadSize"] = narInfo->fileSize;
                     if (showClosureSize)
-                        jsonPath.attr("closureDownloadSize", closureSizes.second);
+                        jsonPath["closureDownloadSize"] = closureSizes.second;
                 }
             }
 
         } catch (InvalidPath &) {
-            jsonPath.attr("path", printStorePath(storePath));
-            jsonPath.attr("valid", false);
+            jsonPath["path"] = printStorePath(storePath);
+            jsonPath["valid"] = false;
         }
     }
+    return jsonList;
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index c8a667c6d..151ec10d6 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -14,6 +14,7 @@
 #include "path-info.hh"
 #include "repair-flag.hh"
 
+#include <nlohmann/json_fwd.hpp>
 #include <atomic>
 #include <limits>
 #include <map>
@@ -68,7 +69,6 @@ struct Derivation;
 class FSAccessor;
 class NarInfoDiskCache;
 class Store;
-class JSONPlaceholder;
 
 
 enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
@@ -512,7 +512,7 @@ public:
        variable elements such as the registration time are
        included. If ‘showClosureSize’ is true, the closure size of
        each path is included. */
-    void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
+    nlohmann::json pathInfoToJSON(const StorePathSet & storePaths,
         bool includeImpureInfo, bool showClosureSize,
         Base hashBase = Base32,
         AllowInvalidFlag allowInvalid = DisallowInvalid);
diff --git a/src/libutil/json.cc b/src/libutil/json.cc
deleted file mode 100644
index 2f9e97ff5..000000000
--- a/src/libutil/json.cc
+++ /dev/null
@@ -1,203 +0,0 @@
-#include "json.hh"
-
-#include <iomanip>
-#include <cstdint>
-#include <cstring>
-
-namespace nix {
-
-template<>
-void toJSON<std::string_view>(std::ostream & str, const std::string_view & s)
-{
-    constexpr size_t BUF_SIZE = 4096;
-    char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts
-    size_t bufPos = 0;
-
-    const auto flush = [&] {
-        str.write(buf, bufPos);
-        bufPos = 0;
-    };
-    const auto put = [&] (char c) {
-        buf[bufPos++] = c;
-    };
-
-    put('"');
-    for (auto i = s.begin(); i != s.end(); i++) {
-        if (bufPos >= BUF_SIZE) flush();
-        if (*i == '\"' || *i == '\\') { put('\\'); put(*i); }
-        else if (*i == '\n') { put('\\'); put('n'); }
-        else if (*i == '\r') { put('\\'); put('r'); }
-        else if (*i == '\t') { put('\\'); put('t'); }
-        else if (*i >= 0 && *i < 32) {
-            const char hex[17] = "0123456789abcdef";
-            put('\\');
-            put('u');
-            put(hex[(uint16_t(*i) >> 12) & 0xf]);
-            put(hex[(uint16_t(*i) >>  8) & 0xf]);
-            put(hex[(uint16_t(*i) >>  4) & 0xf]);
-            put(hex[(uint16_t(*i) >>  0) & 0xf]);
-        }
-        else put(*i);
-    }
-    put('"');
-    flush();
-}
-
-void toJSON(std::ostream & str, const char * s)
-{
-    if (!s) str << "null"; else toJSON(str, std::string_view(s));
-}
-
-template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; }
-template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; }
-template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; }
-template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; }
-template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; }
-template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
-template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
-template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
-template<> void toJSON<std::string>(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); }
-
-template<> void toJSON<bool>(std::ostream & str, const bool & b)
-{
-    str << (b ? "true" : "false");
-}
-
-template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b)
-{
-    str << "null";
-}
-
-JSONWriter::JSONWriter(std::ostream & str, bool indent)
-    : state(new JSONState(str, indent))
-{
-    state->stack++;
-}
-
-JSONWriter::JSONWriter(JSONState * state)
-    : state(state)
-{
-    state->stack++;
-}
-
-JSONWriter::~JSONWriter()
-{
-    if (state) {
-        assertActive();
-        state->stack--;
-        if (state->stack == 0) delete state;
-    }
-}
-
-void JSONWriter::comma()
-{
-    assertActive();
-    if (first) {
-        first = false;
-    } else {
-        state->str << ',';
-    }
-    if (state->indent) indent();
-}
-
-void JSONWriter::indent()
-{
-    state->str << '\n' << std::string(state->depth * 2, ' ');
-}
-
-void JSONList::open()
-{
-    state->depth++;
-    state->str << '[';
-}
-
-JSONList::~JSONList()
-{
-    state->depth--;
-    if (state->indent && !first) indent();
-    state->str << "]";
-}
-
-JSONList JSONList::list()
-{
-    comma();
-    return JSONList(state);
-}
-
-JSONObject JSONList::object()
-{
-    comma();
-    return JSONObject(state);
-}
-
-JSONPlaceholder JSONList::placeholder()
-{
-    comma();
-    return JSONPlaceholder(state);
-}
-
-void JSONObject::open()
-{
-    state->depth++;
-    state->str << '{';
-}
-
-JSONObject::~JSONObject()
-{
-    if (state) {
-        state->depth--;
-        if (state->indent && !first) indent();
-        state->str << "}";
-    }
-}
-
-void JSONObject::attr(std::string_view s)
-{
-    comma();
-    toJSON(state->str, s);
-    state->str << ':';
-    if (state->indent) state->str << ' ';
-}
-
-JSONList JSONObject::list(std::string_view name)
-{
-    attr(name);
-    return JSONList(state);
-}
-
-JSONObject JSONObject::object(std::string_view name)
-{
-    attr(name);
-    return JSONObject(state);
-}
-
-JSONPlaceholder JSONObject::placeholder(std::string_view name)
-{
-    attr(name);
-    return JSONPlaceholder(state);
-}
-
-JSONList JSONPlaceholder::list()
-{
-    assertValid();
-    first = false;
-    return JSONList(state);
-}
-
-JSONObject JSONPlaceholder::object()
-{
-    assertValid();
-    first = false;
-    return JSONObject(state);
-}
-
-JSONPlaceholder::~JSONPlaceholder()
-{
-    if (first) {
-        assert(std::uncaught_exceptions());
-        if (state->stack != 0)
-            write(nullptr);
-    }
-}
-
-}
diff --git a/src/libutil/json.hh b/src/libutil/json.hh
deleted file mode 100644
index 3790b1a2e..000000000
--- a/src/libutil/json.hh
+++ /dev/null
@@ -1,185 +0,0 @@
-#pragma once
-
-#include <iostream>
-#include <vector>
-#include <cassert>
-
-namespace nix {
-
-void toJSON(std::ostream & str, const char * s);
-
-template<typename T>
-void toJSON(std::ostream & str, const T & n);
-
-class JSONWriter
-{
-protected:
-
-    struct JSONState
-    {
-        std::ostream & str;
-        bool indent;
-        size_t depth = 0;
-        size_t stack = 0;
-        JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { }
-        ~JSONState()
-        {
-            assert(stack == 0);
-        }
-    };
-
-    JSONState * state;
-
-    bool first = true;
-
-    JSONWriter(std::ostream & str, bool indent);
-
-    JSONWriter(JSONState * state);
-
-    ~JSONWriter();
-
-    void assertActive()
-    {
-        assert(state->stack != 0);
-    }
-
-    void comma();
-
-    void indent();
-};
-
-class JSONObject;
-class JSONPlaceholder;
-
-class JSONList : JSONWriter
-{
-private:
-
-    friend class JSONObject;
-    friend class JSONPlaceholder;
-
-    void open();
-
-    JSONList(JSONState * state)
-        : JSONWriter(state)
-    {
-        open();
-    }
-
-public:
-
-    JSONList(std::ostream & str, bool indent = false)
-        : JSONWriter(str, indent)
-    {
-        open();
-    }
-
-    ~JSONList();
-
-    template<typename T>
-    JSONList & elem(const T & v)
-    {
-        comma();
-        toJSON(state->str, v);
-        return *this;
-    }
-
-    JSONList list();
-
-    JSONObject object();
-
-    JSONPlaceholder placeholder();
-};
-
-class JSONObject : JSONWriter
-{
-private:
-
-    friend class JSONList;
-    friend class JSONPlaceholder;
-
-    void open();
-
-    JSONObject(JSONState * state)
-        : JSONWriter(state)
-    {
-        open();
-    }
-
-    void attr(std::string_view s);
-
-public:
-
-    JSONObject(std::ostream & str, bool indent = false)
-        : JSONWriter(str, indent)
-    {
-        open();
-    }
-
-    JSONObject(const JSONObject & obj) = delete;
-
-    JSONObject(JSONObject && obj)
-        : JSONWriter(obj.state)
-    {
-        obj.state = 0;
-    }
-
-    ~JSONObject();
-
-    template<typename T>
-    JSONObject & attr(std::string_view name, const T & v)
-    {
-        attr(name);
-        toJSON(state->str, v);
-        return *this;
-    }
-
-    JSONList list(std::string_view name);
-
-    JSONObject object(std::string_view name);
-
-    JSONPlaceholder placeholder(std::string_view name);
-};
-
-class JSONPlaceholder : JSONWriter
-{
-
-private:
-
-    friend class JSONList;
-    friend class JSONObject;
-
-    JSONPlaceholder(JSONState * state)
-        : JSONWriter(state)
-    {
-    }
-
-    void assertValid()
-    {
-        assertActive();
-        assert(first);
-    }
-
-public:
-
-    JSONPlaceholder(std::ostream & str, bool indent = false)
-        : JSONWriter(str, indent)
-    {
-    }
-
-    ~JSONPlaceholder();
-
-    template<typename T>
-    void write(const T & v)
-    {
-        assertValid();
-        first = false;
-        toJSON(state->str, v);
-    }
-
-    JSONList list();
-
-    JSONObject object();
-};
-
-}
diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc
deleted file mode 100644
index 156286999..000000000
--- a/src/libutil/tests/json.cc
+++ /dev/null
@@ -1,193 +0,0 @@
-#include "json.hh"
-#include <gtest/gtest.h>
-#include <sstream>
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * toJSON
-     * --------------------------------------------------------------------------*/
-
-    TEST(toJSON, quotesCharPtr) {
-        const char* input = "test";
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "\"test\"");
-    }
-
-    TEST(toJSON, quotesStdString) {
-        std::string input = "test";
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "\"test\"");
-    }
-
-    TEST(toJSON, convertsNullptrtoNull) {
-        auto input = nullptr;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "null");
-    }
-
-    TEST(toJSON, convertsNullToNull) {
-        const char* input = 0;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "null");
-    }
-
-
-    TEST(toJSON, convertsFloat) {
-        auto input = 1.024f;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "1.024");
-    }
-
-    TEST(toJSON, convertsDouble) {
-        const double input = 1.024;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "1.024");
-    }
-
-    TEST(toJSON, convertsBool) {
-        auto input = false;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "false");
-    }
-
-    TEST(toJSON, quotesTab) {
-        std::stringstream out;
-        toJSON(out, "\t");
-
-        ASSERT_EQ(out.str(), "\"\\t\"");
-    }
-
-    TEST(toJSON, quotesNewline) {
-        std::stringstream out;
-        toJSON(out, "\n");
-
-        ASSERT_EQ(out.str(), "\"\\n\"");
-    }
-
-    TEST(toJSON, quotesCreturn) {
-        std::stringstream out;
-        toJSON(out, "\r");
-
-        ASSERT_EQ(out.str(), "\"\\r\"");
-    }
-
-    TEST(toJSON, quotesCreturnNewLine) {
-        std::stringstream out;
-        toJSON(out, "\r\n");
-
-        ASSERT_EQ(out.str(), "\"\\r\\n\"");
-    }
-
-    TEST(toJSON, quotesDoublequotes) {
-        std::stringstream out;
-        toJSON(out, "\"");
-
-        ASSERT_EQ(out.str(), "\"\\\"\"");
-    }
-
-    TEST(toJSON, substringEscape) {
-        std::stringstream out;
-        std::string_view s = "foo\t";
-        toJSON(out, s.substr(3));
-
-        ASSERT_EQ(out.str(), "\"\\t\"");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * JSONObject
-     * --------------------------------------------------------------------------*/
-
-    TEST(JSONObject, emptyObject) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-        }
-        ASSERT_EQ(out.str(), "{}");
-    }
-
-    TEST(JSONObject, objectWithList) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-            auto l = t.list("list");
-            l.elem("element");
-        }
-        ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
-    }
-
-    TEST(JSONObject, objectWithListIndent) {
-        std::stringstream out;
-        {
-            JSONObject t(out, true);
-            auto l = t.list("list");
-            l.elem("element");
-        }
-        ASSERT_EQ(out.str(),
-R"#({
-  "list": [
-    "element"
-  ]
-})#");
-    }
-
-    TEST(JSONObject, objectWithPlaceholderAndList) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-            auto l = t.placeholder("list");
-            l.list().elem("element");
-        }
-
-        ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
-    }
-
-    TEST(JSONObject, objectWithPlaceholderAndObject) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-            auto l = t.placeholder("object");
-            l.object().attr("key", "value");
-        }
-
-        ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * JSONList
-     * --------------------------------------------------------------------------*/
-
-    TEST(JSONList, empty) {
-        std::stringstream out;
-        {
-            JSONList l(out);
-        }
-        ASSERT_EQ(out.str(), R"#([])#");
-    }
-
-    TEST(JSONList, withElements) {
-        std::stringstream out;
-        {
-            JSONList l(out);
-            l.elem("one");
-            l.object();
-            l.placeholder().write("three");
-        }
-        ASSERT_EQ(out.str(), R"#(["one",{},"three"])#");
-    }
-}
-
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index fdd66220a..776c5f6db 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -12,7 +12,6 @@
 #include "local-fs-store.hh"
 #include "user-env.hh"
 #include "util.hh"
-#include "json.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
 #include "legacy.hh"
@@ -26,6 +25,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 using std::cout;
@@ -911,43 +911,47 @@ static VersionDiff compareVersionAgainstSet(
 
 static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printMeta)
 {
-    JSONObject topObj(cout, true);
+    using nlohmann::json;
+    json topObj = json::object();
     for (auto & i : elems) {
         try {
             if (i.hasFailed()) continue;
 
-            JSONObject pkgObj = topObj.object(i.attrPath);
 
             auto drvName = DrvName(i.queryName());
-            pkgObj.attr("name", drvName.fullName);
-            pkgObj.attr("pname", drvName.name);
-            pkgObj.attr("version", drvName.version);
-            pkgObj.attr("system", i.querySystem());
-            pkgObj.attr("outputName", i.queryOutputName());
+            json &pkgObj = topObj[i.attrPath];
+            pkgObj = {
+                {"name", drvName.fullName},
+                {"pname", drvName.name},
+                {"version", drvName.version},
+                {"system", i.querySystem()},
+                {"outputName", i.queryOutputName()},
+            };
 
             {
                 DrvInfo::Outputs outputs = i.queryOutputs(printOutPath);
-                JSONObject outputObj = pkgObj.object("outputs");
+                json &outputObj = pkgObj["outputs"];
+                outputObj = json::object();
                 for (auto & j : outputs) {
                     if (j.second)
-                        outputObj.attr(j.first, globals.state->store->printStorePath(*j.second));
+                        outputObj[j.first] = globals.state->store->printStorePath(*j.second);
                     else
-                        outputObj.attr(j.first, nullptr);
+                        outputObj[j.first] = nullptr;
                 }
             }
 
             if (printMeta) {
-                JSONObject metaObj = pkgObj.object("meta");
+                json &metaObj = pkgObj["meta"];
+                metaObj = json::object();
                 StringSet metaNames = i.queryMetaNames();
                 for (auto & j : metaNames) {
                     Value * v = i.queryMeta(j);
                     if (!v) {
                         printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
-                        metaObj.attr(j, nullptr);
+                        metaObj[j] = nullptr;
                     } else {
-                        auto placeholder = metaObj.placeholder(j);
                         PathSet context;
-                        printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context);
+                        metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context);
                     }
                 }
             }
@@ -958,6 +962,7 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
             throw;
         }
     }
+    std::cout << topObj.dump(2);
 }
 
 
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index ddd2790c6..ba82b5772 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -4,10 +4,11 @@
 #include "store-api.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
-#include "json.hh"
 #include "value-to-json.hh"
 #include "progress-bar.hh"
 
+#include <nlohmann/json.hpp>
+
 using namespace nix;
 
 struct CmdEval : MixJSON, InstallableCommand
@@ -115,9 +116,7 @@ struct CmdEval : MixJSON, InstallableCommand
         }
 
         else if (json) {
-            JSONPlaceholder jsonOut(std::cout);
-            printValueAsJSON(*state, true, *v, pos, jsonOut, context, false);
-            std::cout << std::endl;
+            std::cout << printValueAsJSON(*state, true, *v, pos, context, false).dump() << std::endl;
         }
 
         else {
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 3967f1102..336f6723a 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -11,7 +11,6 @@
 #include "attr-path.hh"
 #include "fetchers.hh"
 #include "registry.hh"
-#include "json.hh"
 #include "eval-cache.hh"
 #include "markdown.hh"
 
@@ -21,6 +20,7 @@
 
 using namespace nix;
 using namespace nix::flake;
+using json = nlohmann::json;
 
 class FlakeCommand : virtual Args, public MixFlakeOptions
 {
@@ -917,35 +917,44 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
     {
         auto flake = lockFlake();
 
-        auto jsonRoot = json ? std::optional<JSONObject>(std::cout) : std::nullopt;
-
         StorePathSet sources;
 
         sources.insert(flake.flake.sourceInfo->storePath);
-        if (jsonRoot)
-            jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath));
 
         // FIXME: use graph output, handle cycles.
-        std::function<void(const Node & node, std::optional<JSONObject> & jsonObj)> traverse;
-        traverse = [&](const Node & node, std::optional<JSONObject> & jsonObj)
+        std::function<nlohmann::json(const Node & node)> traverse;
+        traverse = [&](const Node & node)
         {
-            auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional<JSONObject>();
+            nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr);
             for (auto & [inputName, input] : node.inputs) {
                 if (auto inputNode = std::get_if<0>(&input)) {
-                    auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional<JSONObject>();
                     auto storePath =
                         dryRun
                         ? (*inputNode)->lockedRef.input.computeStorePath(*store)
                         : (*inputNode)->lockedRef.input.fetch(store).first.storePath;
-                    if (jsonObj3)
-                        jsonObj3->attr("path", store->printStorePath(storePath));
-                    sources.insert(std::move(storePath));
-                    traverse(**inputNode, jsonObj3);
+                    if (json) {
+                        auto& jsonObj3 = jsonObj2[inputName];
+                        jsonObj3["path"] = store->printStorePath(storePath);
+                        sources.insert(std::move(storePath));
+                        jsonObj3["inputs"] = traverse(**inputNode);
+                    } else {
+                        sources.insert(std::move(storePath));
+                        traverse(**inputNode);
+                    }
                 }
             }
+            return jsonObj2;
         };
 
-        traverse(*flake.lockFile.root, jsonRoot);
+        if (json) {
+            nlohmann::json jsonRoot = {
+                {"path", store->printStorePath(flake.flake.sourceInfo->storePath)},
+                {"inputs", traverse(*flake.lockFile.root)},
+            };
+            std::cout << jsonRoot.dump() << std::endl;
+        } else {
+            traverse(*flake.lockFile.root);
+        }
 
         if (!dryRun && !dstUri.empty()) {
             ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 07554994b..e964b01b3 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -3,7 +3,7 @@
 #include "fs-accessor.hh"
 #include "nar-accessor.hh"
 #include "common-args.hh"
-#include "json.hh"
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -91,10 +91,9 @@ struct MixLs : virtual Args, MixJSON
         if (path == "/") path = "";
 
         if (json) {
-            JSONPlaceholder jsonRoot(std::cout);
             if (showDirectory)
                 throw UsageError("'--directory' is useless with '--json'");
-            listNar(jsonRoot, accessor, path, recursive);
+            std::cout << listNar(accessor, path, recursive);
         } else
             listText(accessor);
     }
diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc
index 34860c38f..f2e4cefbe 100644
--- a/src/nix/make-content-addressed.cc
+++ b/src/nix/make-content-addressed.cc
@@ -2,10 +2,13 @@
 #include "store-api.hh"
 #include "make-content-addressed.hh"
 #include "common-args.hh"
-#include "json.hh"
+
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
+using nlohmann::json;
+
 struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
 {
     CmdMakeContentAddressed()
@@ -25,6 +28,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
           ;
     }
 
+    using StorePathsCommand::run;
     void run(ref<Store> srcStore, StorePaths && storePaths) override
     {
         auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
@@ -33,13 +37,13 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
             StorePathSet(storePaths.begin(), storePaths.end()));
 
         if (json) {
-            JSONObject jsonRoot(std::cout);
-            JSONObject jsonRewrites(jsonRoot.object("rewrites"));
+            nlohmann::json jsonRewrites = json::object();
             for (auto & path : storePaths) {
                 auto i = remappings.find(path);
                 assert(i != remappings.end());
-                jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second));
+                jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second);
             }
+            std::cout << json::object({"rewrites", jsonRewrites}).dump();
         } else {
             for (auto & path : storePaths) {
                 auto i = remappings.find(path);
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index d690fe594..613c5b191 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -1,12 +1,13 @@
 #include "command.hh"
 #include "shared.hh"
 #include "store-api.hh"
-#include "json.hh"
 #include "common-args.hh"
 
 #include <algorithm>
 #include <array>
 
+#include <nlohmann/json.hpp>
+
 using namespace nix;
 
 struct CmdPathInfo : StorePathsCommand, MixJSON
@@ -86,11 +87,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             pathLen = std::max(pathLen, store->printStorePath(storePath).size());
 
         if (json) {
-            JSONPlaceholder jsonRoot(std::cout);
-            store->pathInfoToJSON(jsonRoot,
+            std::cout << store->pathInfoToJSON(
                 // FIXME: preserve order?
                 StorePathSet(storePaths.begin(), storePaths.end()),
-                true, showClosureSize, SRI, AllowInvalid);
+                true, showClosureSize, SRI, AllowInvalid).dump();
         }
 
         else {
diff --git a/src/nix/search.cc b/src/nix/search.cc
index bdd45cbed..d2a31607d 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -5,7 +5,6 @@
 #include "names.hh"
 #include "get-drvs.hh"
 #include "common-args.hh"
-#include "json.hh"
 #include "shared.hh"
 #include "eval-cache.hh"
 #include "attr-path.hh"
@@ -13,8 +12,10 @@
 
 #include <regex>
 #include <fstream>
+#include <nlohmann/json.hpp>
 
 using namespace nix;
+using json = nlohmann::json;
 
 std::string wrap(std::string prefix, std::string s)
 {
@@ -84,7 +85,8 @@ struct CmdSearch : InstallableCommand, MixJSON
 
         auto state = getEvalState();
 
-        auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
+        std::optional<nlohmann::json> jsonOut;
+        if (json) jsonOut = json::object();
 
         uint64_t results = 0;
 
@@ -151,10 +153,11 @@ struct CmdSearch : InstallableCommand, MixJSON
                     {
                         results++;
                         if (json) {
-                            auto jsonElem = jsonOut->object(attrPath2);
-                            jsonElem.attr("pname", name.name);
-                            jsonElem.attr("version", name.version);
-                            jsonElem.attr("description", description);
+                            (*jsonOut)[attrPath2] = {
+                                {"pname", name.name},
+                                {"version", name.version},
+                                {"description", description},
+                            };
                         } else {
                             auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m");
                             if (results > 1) logger->cout("");
@@ -193,6 +196,10 @@ struct CmdSearch : InstallableCommand, MixJSON
         for (auto & cursor : installable->getCursors(*state))
             visit(*cursor, cursor->getAttrPath(), true);
 
+        if (json) {
+            std::cout << jsonOut->dump() << std::endl;
+        }
+
         if (!json && !results)
             throw Error("no results for the given search term(s)!");
     }
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index fb46b4dbf..af2e676a4 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -5,10 +5,11 @@
 #include "common-args.hh"
 #include "store-api.hh"
 #include "archive.hh"
-#include "json.hh"
 #include "derivations.hh"
+#include <nlohmann/json.hpp>
 
 using namespace nix;
+using json = nlohmann::json;
 
 struct CmdShowDerivation : InstallablesCommand
 {
@@ -48,77 +49,63 @@ struct CmdShowDerivation : InstallablesCommand
             drvPaths = std::move(closure);
         }
 
-        {
-
-        JSONObject jsonRoot(std::cout, true);
+        json jsonRoot = json::object();
 
         for (auto & drvPath : drvPaths) {
             if (!drvPath.isDerivation()) continue;
 
-            auto drvObj(jsonRoot.object(store->printStorePath(drvPath)));
+            json& drvObj = jsonRoot[store->printStorePath(drvPath)];
 
             auto drv = store->readDerivation(drvPath);
 
             {
-                auto outputsObj(drvObj.object("outputs"));
+                json& outputsObj = drvObj["outputs"];
+                outputsObj = json::object();
                 for (auto & [_outputName, output] : drv.outputs) {
                     auto & outputName = _outputName; // work around clang bug
-                    auto outputObj { outputsObj.object(outputName) };
+                    auto& outputObj = outputsObj[outputName];
+                    outputObj = json::object();
                     std::visit(overloaded {
                         [&](const DerivationOutput::InputAddressed & doi) {
-                            outputObj.attr("path", store->printStorePath(doi.path));
+                            outputObj["path"] = store->printStorePath(doi.path);
                         },
                         [&](const DerivationOutput::CAFixed & dof) {
-                            outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName)));
-                            outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
-                            outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
+                            outputObj["path"] = store->printStorePath(dof.path(*store, drv.name, outputName));
+                            outputObj["hashAlgo"] = dof.hash.printMethodAlgo();
+                            outputObj["hash"] = dof.hash.hash.to_string(Base16, false);
                         },
                         [&](const DerivationOutput::CAFloating & dof) {
-                            outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+                            outputObj["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType);
                         },
                         [&](const DerivationOutput::Deferred &) {},
                         [&](const DerivationOutput::Impure & doi) {
-                            outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
-                            outputObj.attr("impure", true);
+                            outputObj["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType);
+                            outputObj["impure"] = true;
                         },
                     }, output.raw());
                 }
             }
 
             {
-                auto inputsList(drvObj.list("inputSrcs"));
+                auto& inputsList = drvObj["inputSrcs"];
+                inputsList = json::array();
                 for (auto & input : drv.inputSrcs)
-                    inputsList.elem(store->printStorePath(input));
+                    inputsList.emplace_back(store->printStorePath(input));
             }
 
             {
-                auto inputDrvsObj(drvObj.object("inputDrvs"));
-                for (auto & input : drv.inputDrvs) {
-                    auto inputList(inputDrvsObj.list(store->printStorePath(input.first)));
-                    for (auto & outputId : input.second)
-                        inputList.elem(outputId);
-                }
+                auto& inputDrvsObj = drvObj["inputDrvs"];
+                inputDrvsObj = json::object();
+                for (auto & input : drv.inputDrvs)
+                    inputDrvsObj[store->printStorePath(input.first)] = input.second;
             }
 
-            drvObj.attr("system", drv.platform);
-            drvObj.attr("builder", drv.builder);
-
-            {
-                auto argsList(drvObj.list("args"));
-                for (auto & arg : drv.args)
-                    argsList.elem(arg);
-            }
-
-            {
-                auto envObj(drvObj.object("env"));
-                for (auto & var : drv.env)
-                    envObj.attr(var.first, var.second);
-            }
+            drvObj["system"] = drv.platform;
+            drvObj["builder"] = drv.builder;
+            drvObj["args"] = drv.args;
+            drvObj["env"] = drv.env;
         }
-
-        }
-
-        std::cout << "\n";
+        std::cout << jsonRoot.dump(2) << std::endl;
     }
 };
 

From f423d4425f6573206045e9626812002906d9493d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 17 Nov 2022 11:56:45 +0100
Subject: [PATCH 076/112] Fix segfault in unprivileged mode

---
 src/libstore/build/local-derivation-goal.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 070ae53f3..f92280aa1 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -92,8 +92,8 @@ struct LocalDerivationGoal : public DerivationGoal
        result. */
     std::map<Path, ValidPathInfo> prevInfos;
 
-    uid_t sandboxUid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 100  : 0) : buildUser->getGID(); }
+    uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
+    gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100  : 0) : buildUser->getGID(); }
 
     const static Path homeDir;
 

From f1ab082ac4f589a36a9eb0cd98d1cc235eedc419 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Nov 2022 09:37:11 +0100
Subject: [PATCH 077/112] createTempDir(): Use std::atomic

---
 src/libutil/filesystem.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc
index 403389e60..3a732cff8 100644
--- a/src/libutil/filesystem.cc
+++ b/src/libutil/filesystem.cc
@@ -1,5 +1,6 @@
 #include <sys/time.h>
 #include <filesystem>
+#include <atomic>
 
 #include "finally.hh"
 #include "util.hh"
@@ -10,7 +11,7 @@ namespace fs = std::filesystem;
 namespace nix {
 
 static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
-    int & counter)
+    std::atomic<unsigned int> & counter)
 {
     tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
     if (includePid)
@@ -22,9 +23,9 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
 Path createTempDir(const Path & tmpRoot, const Path & prefix,
     bool includePid, bool useGlobalCounter, mode_t mode)
 {
-    static int globalCounter = 0;
-    int localCounter = 0;
-    int & counter(useGlobalCounter ? globalCounter : localCounter);
+    static std::atomic<unsigned int> globalCounter = 0;
+    std::atomic<unsigned int> localCounter = 0;
+    auto & counter(useGlobalCounter ? globalCounter : localCounter);
 
     while (1) {
         checkInterrupt();

From 128910ba23f586ba1765a137ecff23cfd22cff89 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Nov 2022 10:39:28 +0100
Subject: [PATCH 078/112] Separate cgroup support from auto-uid-allocation

The new experimental feature 'cgroups' enables the use of cgroups for
all builds. This allows better containment and enables setting
resource limits and getting some build stats.
---
 src/libstore/build/local-derivation-goal.cc | 114 ++++++++++++++------
 src/libstore/build/local-derivation-goal.hh |   7 ++
 src/libstore/lock.cc                        |  71 ------------
 src/libstore/lock.hh                        |   7 --
 src/libutil/experimental-features.cc        |   1 +
 src/libutil/experimental-features.hh        |   1 +
 6 files changed, 93 insertions(+), 108 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index e652c425c..2d1e093ca 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -15,6 +15,7 @@
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "json-utils.hh"
+#include "cgroup.hh"
 
 #include <regex>
 #include <queue>
@@ -129,26 +130,36 @@ void LocalDerivationGoal::killChild()
     if (pid != -1) {
         worker.childTerminated(this);
 
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
+        /* If we're using a build user, then there is a tricky race
+           condition: if we kill the build user before the child has
+           done its setuid() to the build user uid, then it won't be
+           killed, and we'll potentially lock up in pid.wait().  So
+           also send a conventional kill to the child. */
+        ::kill(-pid, SIGKILL); /* ignore the result */
 
-        assert(pid == -1);
+        killSandbox();
+
+        pid.wait();
     }
 
     DerivationGoal::killChild();
 }
 
 
+void LocalDerivationGoal::killSandbox()
+{
+    if (cgroup) {
+        destroyCgroup(*cgroup);
+    }
+
+    else if (buildUser) {
+        auto uid = buildUser->getUID();
+        assert(uid != 0);
+        killUser(uid);
+    }
+}
+
+
 void LocalDerivationGoal::tryLocalBuild() {
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs) {
@@ -169,10 +180,6 @@ void LocalDerivationGoal::tryLocalBuild() {
             worker.waitForAWhile(shared_from_this());
             return;
         }
-
-        /* Make sure that no other processes are executing under this
-           uid. */
-        buildUser->kill();
     }
 
     actLock.reset();
@@ -263,7 +270,7 @@ void LocalDerivationGoal::cleanupPostChildKill()
        malicious user from leaving behind a process that keeps files
        open and modifies them after they have been chown'ed to
        root. */
-    if (buildUser) buildUser->kill();
+    killSandbox();
 
     /* Terminate the recursive Nix daemon. */
     stopDaemon();
@@ -356,6 +363,55 @@ static void linkOrCopy(const Path & from, const Path & to)
 
 void LocalDerivationGoal::startBuilder()
 {
+    if ((buildUser && buildUser->getUIDCount() != 1)
+        || settings.isExperimentalFeatureEnabled(Xp::Cgroups))
+    {
+        #if __linux__
+        auto ourCgroups = getCgroups("/proc/self/cgroup");
+        auto ourCgroup = ourCgroups[""];
+        if (ourCgroup == "")
+            throw Error("cannot determine cgroup name from /proc/self/cgroup");
+
+        auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup);
+
+        if (!pathExists(ourCgroupPath))
+            throw Error("expected cgroup directory '%s'", ourCgroupPath);
+
+        static std::atomic<unsigned int> counter{0};
+
+        cgroup = buildUser
+            ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID())
+            : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++);
+
+        debug("using cgroup '%s'", *cgroup);
+
+        /* When using a build user, record the cgroup we used for that
+           user so that if we got interrupted previously, we can kill
+           any left-over cgroup first. */
+        if (buildUser) {
+            auto cgroupsDir = settings.nixStateDir + "/cgroups";
+            createDirs(cgroupsDir);
+
+            auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID());
+
+            if (pathExists(cgroupFile)) {
+                auto prevCgroup = readFile(cgroupFile);
+                destroyCgroup(prevCgroup);
+            }
+
+            writeFile(cgroupFile, *cgroup);
+        }
+
+        #else
+        throw Error("cgroups are not supported on this platform");
+        #endif
+    }
+
+    /* Make sure that no other processes are executing under the
+       sandbox uids. This must be done before any chownToBuilder()
+       calls. */
+    killSandbox();
+
     /* Right platform? */
     if (!parsedDrv->canBuildLocally(worker.store))
         throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
@@ -646,13 +702,13 @@ void LocalDerivationGoal::startBuilder()
                 dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
         }
 
-        if (buildUser) {
-            if (auto cgroup = buildUser->getCgroup()) {
-                chownToBuilder(*cgroup);
-                chownToBuilder(*cgroup + "/cgroup.procs");
-                chownToBuilder(*cgroup + "/cgroup.threads");
-                //chownToBuilder(*cgroup + "/cgroup.subtree_control");
-            }
+        if (cgroup) {
+            if (mkdir(cgroup->c_str(), 0755) != 0)
+                throw SysError("creating cgroup '%s'", *cgroup);
+            chownToBuilder(*cgroup);
+            chownToBuilder(*cgroup + "/cgroup.procs");
+            chownToBuilder(*cgroup + "/cgroup.threads");
+            //chownToBuilder(*cgroup + "/cgroup.subtree_control");
         }
 
 #else
@@ -965,10 +1021,8 @@ void LocalDerivationGoal::startBuilder()
         }
 
         /* Move the child into its own cgroup. */
-        if (buildUser) {
-            if (auto cgroup = buildUser->getCgroup())
-                writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
-        }
+        if (cgroup)
+            writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
 
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
@@ -1838,7 +1892,7 @@ void LocalDerivationGoal::runChild()
             /* Unshare the cgroup namespace. This means
                /proc/self/cgroup will show the child's cgroup as '/'
                rather than whatever it is in the parent. */
-            if (buildUser && buildUser->getUIDCount() != 1 && unshare(CLONE_NEWCGROUP) == -1)
+            if (cgroup && unshare(CLONE_NEWCGROUP) == -1)
                 throw SysError("unsharing cgroup namespace");
 
             /* Do the chroot(). */
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index f92280aa1..1ec6b3649 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -15,6 +15,9 @@ struct LocalDerivationGoal : public DerivationGoal
     /* The process ID of the builder. */
     Pid pid;
 
+    /* The cgroup of the builder, if any. */
+    std::optional<Path> cgroup;
+
     /* The temporary directory. */
     Path tmpDir;
 
@@ -197,6 +200,10 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Forcibly kill the child process, if any. */
     void killChild() override;
 
+    /* Kill any processes running under the build user UID or in the
+       cgroup of the build. */
+    void killSandbox();
+
     /* Create alternative path calculated from but distinct from the
        input, so we can avoid overwriting outputs (or other store paths)
        that already exist. */
diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index f9892bb91..4fad3bfd2 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -1,7 +1,6 @@
 #include "lock.hh"
 #include "globals.hh"
 #include "pathlocks.hh"
-#include "cgroup.hh"
 
 #include <pwd.h>
 #include <grp.h>
@@ -15,11 +14,6 @@ struct SimpleUserLock : UserLock
     gid_t gid;
     std::vector<gid_t> supplementaryGIDs;
 
-    void kill() override
-    {
-        killUser(uid);
-    }
-
     uid_t getUID() override { assert(uid); return uid; }
     uid_t getUIDCount() override { return 1; }
     gid_t getGID() override { assert(gid); return gid; }
@@ -116,32 +110,6 @@ struct AutoUserLock : UserLock
     AutoCloseFD fdUserLock;
     uid_t firstUid = 0;
     uid_t nrIds = 1;
-    #if __linux__
-    std::optional<Path> cgroup;
-    #endif
-
-    ~AutoUserLock()
-    {
-        #if __linux__
-        // Get rid of our cgroup, ignoring errors.
-        if (cgroup) rmdir(cgroup->c_str());
-        #endif
-    }
-
-    void kill() override
-    {
-        #if __linux__
-        if (cgroup) {
-            destroyCgroup(*cgroup);
-            if (mkdir(cgroup->c_str(), 0755) == -1)
-                throw SysError("creating cgroup '%s'", *cgroup);
-        } else
-        #endif
-        {
-            assert(firstUid);
-            killUser(firstUid);
-        }
-    }
 
     uid_t getUID() override { assert(firstUid); return firstUid; }
 
@@ -183,55 +151,16 @@ struct AutoUserLock : UserLock
                 throw SysError("opening user lock '%s'", fnUserLock);
 
             if (lockFile(fd.get(), ltWrite, false)) {
-                auto s = drainFD(fd.get());
-
-                #if __linux__
-                if (s != "") {
-                    /* Kill the old cgroup, to ensure there are no
-                       processes left over from an interrupted build. */
-                    destroyCgroup(s);
-                }
-                #endif
-
-                if (ftruncate(fd.get(), 0) == -1)
-                    throw Error("truncating user lock");
-
                 auto lock = std::make_unique<AutoUserLock>();
                 lock->fdUserLock = std::move(fd);
                 lock->firstUid = settings.startId + i * maxIdsPerBuild;
                 lock->nrIds = nrIds;
-
-                #if __linux__
-                if (nrIds > 1) {
-                    auto ourCgroups = getCgroups("/proc/self/cgroup");
-                    auto ourCgroup = ourCgroups[""];
-                    if (ourCgroup == "")
-                        throw Error("cannot determine cgroup name from /proc/self/cgroup");
-
-                    auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup);
-
-                    if (!pathExists(ourCgroupPath))
-                        throw Error("expected cgroup directory '%s'", ourCgroupPath);
-
-                    lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid);
-
-                    /* Record the cgroup in the lock file. This ensures that
-                       if we subsequently get executed under a different parent
-                       cgroup, we kill the previous cgroup first. */
-                    writeFull(lock->fdUserLock.get(), *lock->cgroup);
-                }
-                #endif
-
                 return lock;
             }
         }
 
         return nullptr;
     }
-
-    #if __linux__
-    std::optional<Path> getCgroup() override { return cgroup; }
-    #endif
 };
 
 std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds)
diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh
index b5536408c..e7ceefab8 100644
--- a/src/libstore/lock.hh
+++ b/src/libstore/lock.hh
@@ -27,13 +27,6 @@ struct UserLock
     virtual gid_t getGID() = 0;
 
     virtual std::vector<gid_t> getSupplementaryGIDs() = 0;
-
-    /* Kill any processes currently executing as this user. */
-    virtual void kill() = 0;
-
-    #if __linux__
-    virtual std::optional<Path> getCgroup() { return {}; };
-    #endif
 };
 
 /* Acquire a user lock for a UID range of size `nrIds`. Note that this
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 0f05f3752..e0902971e 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -15,6 +15,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::FetchClosure, "fetch-closure" },
     { Xp::ReplFlake, "repl-flake" },
     { Xp::AutoAllocateUids, "auto-allocate-uids" },
+    { Xp::Cgroups, "cgroups" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index cf0c06eac..af775feb0 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -24,6 +24,7 @@ enum struct ExperimentalFeature
     FetchClosure,
     ReplFlake,
     AutoAllocateUids,
+    Cgroups,
 };
 
 /**

From 20f66c6889aa9d907feee4946702d655b6bd796f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Nov 2022 13:40:48 +0100
Subject: [PATCH 079/112] Indentation

---
 src/libcmd/installables.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index e097f23b3..5945a2578 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -927,7 +927,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
 
     case Realise::Outputs: {
         if (settings.printMissing)
-          printMissing(store, pathsToBuild, lvlInfo);
+            printMissing(store, pathsToBuild, lvlInfo);
 
         for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
             if (!buildResult.success())

From fa68eb367e79297bb1c0451cd92ad18a06edce96 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Nov 2022 13:40:59 +0100
Subject: [PATCH 080/112] Get CPU stats from the cgroup

---
 src/libstore/build-result.hh                |  5 ++-
 src/libstore/build/derivation-goal.cc       |  8 +++++
 src/libstore/build/local-derivation-goal.cc | 14 +++++---
 src/libstore/build/local-derivation-goal.hh |  2 +-
 src/libstore/cgroup.cc                      | 39 +++++++++++++++++++--
 src/libstore/cgroup.hh                      | 14 +++++++-
 6 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh
index 24fb1f763..a5749cf33 100644
--- a/src/libstore/build-result.hh
+++ b/src/libstore/build-result.hh
@@ -5,7 +5,7 @@
 
 #include <string>
 #include <chrono>
-
+#include <optional>
 
 namespace nix {
 
@@ -78,6 +78,9 @@ struct BuildResult
        was repeated). */
     time_t startTime = 0, stopTime = 0;
 
+    /* User and system CPU time the build took. */
+    std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
+
     bool success()
     {
         return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 41d2e2a1c..9bc3dc742 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -869,6 +869,14 @@ void DerivationGoal::buildDone()
 
     cleanupPostChildKill();
 
+    if (buildResult.cpuUser && buildResult.cpuSystem) {
+        debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs",
+            worker.store.printStorePath(drvPath),
+            status,
+            ((double) buildResult.cpuUser->count()) / 1000000,
+            ((double) buildResult.cpuSystem->count()) / 1000000);
+    }
+
     bool diskFull = false;
 
     try {
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 2d1e093ca..f273ebe8a 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -137,7 +137,7 @@ void LocalDerivationGoal::killChild()
            also send a conventional kill to the child. */
         ::kill(-pid, SIGKILL); /* ignore the result */
 
-        killSandbox();
+        killSandbox(true);
 
         pid.wait();
     }
@@ -146,10 +146,14 @@ void LocalDerivationGoal::killChild()
 }
 
 
-void LocalDerivationGoal::killSandbox()
+void LocalDerivationGoal::killSandbox(bool getStats)
 {
     if (cgroup) {
-        destroyCgroup(*cgroup);
+        auto stats = destroyCgroup(*cgroup);
+        if (getStats) {
+            buildResult.cpuUser = stats.cpuUser;
+            buildResult.cpuSystem = stats.cpuSystem;
+        }
     }
 
     else if (buildUser) {
@@ -270,7 +274,7 @@ void LocalDerivationGoal::cleanupPostChildKill()
        malicious user from leaving behind a process that keeps files
        open and modifies them after they have been chown'ed to
        root. */
-    killSandbox();
+    killSandbox(true);
 
     /* Terminate the recursive Nix daemon. */
     stopDaemon();
@@ -410,7 +414,7 @@ void LocalDerivationGoal::startBuilder()
     /* Make sure that no other processes are executing under the
        sandbox uids. This must be done before any chownToBuilder()
        calls. */
-    killSandbox();
+    killSandbox(false);
 
     /* Right platform? */
     if (!parsedDrv->canBuildLocally(worker.store))
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 1ec6b3649..34c4e9187 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -202,7 +202,7 @@ struct LocalDerivationGoal : public DerivationGoal
 
     /* Kill any processes running under the build user UID or in the
        cgroup of the build. */
-    void killSandbox();
+    void killSandbox(bool getStats);
 
     /* Create alternative path calculated from but distinct from the
        input, so we can avoid overwriting outputs (or other store paths)
diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc
index 56e980be3..2a485f0f9 100644
--- a/src/libstore/cgroup.cc
+++ b/src/libstore/cgroup.cc
@@ -31,13 +31,16 @@ std::map<std::string, std::string> getCgroups(const Path & cgroupFile)
     return cgroups;
 }
 
-void destroyCgroup(const Path & cgroup)
+static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats)
 {
-    if (!pathExists(cgroup)) return;
+    if (!pathExists(cgroup)) return {};
+
+    if (!pathExists(cgroup + "/cgroup.procs"))
+        throw Error("'%s' is not a cgroup", cgroup);
 
     for (auto & entry : readDirectory(cgroup)) {
         if (entry.type != DT_DIR) continue;
-        destroyCgroup(cgroup + "/" + entry.name);
+        destroyCgroup(cgroup + "/" + entry.name, false);
     }
 
     int round = 1;
@@ -79,8 +82,38 @@ void destroyCgroup(const Path & cgroup)
         round++;
     }
 
+    CgroupStats stats;
+
+    if (returnStats) {
+        auto cpustatPath = cgroup + "/cpu.stat";
+
+        if (pathExists(cpustatPath)) {
+            for (auto & line : tokenizeString<std::vector<std::string>>(readFile(cpustatPath), "\n")) {
+                std::string_view userPrefix = "user_usec ";
+                if (hasPrefix(line, userPrefix)) {
+                    auto n = string2Int<uint64_t>(line.substr(userPrefix.size()));
+                    if (n) stats.cpuUser = std::chrono::microseconds(*n);
+                }
+
+                std::string_view systemPrefix = "system_usec ";
+                if (hasPrefix(line, systemPrefix)) {
+                    auto n = string2Int<uint64_t>(line.substr(systemPrefix.size()));
+                    if (n) stats.cpuSystem = std::chrono::microseconds(*n);
+                }
+            }
+        }
+
+    }
+
     if (rmdir(cgroup.c_str()) == -1)
         throw SysError("deleting cgroup '%s'", cgroup);
+
+    return stats;
+}
+
+CgroupStats destroyCgroup(const Path & cgroup)
+{
+    return destroyCgroup(cgroup, true);
 }
 
 }
diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh
index dc6758957..3ead4735f 100644
--- a/src/libstore/cgroup.hh
+++ b/src/libstore/cgroup.hh
@@ -2,13 +2,25 @@
 
 #if __linux__
 
+#include <chrono>
+#include <optional>
+
 #include "types.hh"
 
 namespace nix {
 
 std::map<std::string, std::string> getCgroups(const Path & cgroupFile);
 
-void destroyCgroup(const Path & cgroup);
+struct CgroupStats
+{
+    std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
+};
+
+/* Destroy the cgroup denoted by 'path'. The postcondition is that
+   'path' does not exist, and thus any processes in the cgroup have
+   been killed. Also return statistics from the cgroup just before
+   destruction. */
+CgroupStats destroyCgroup(const Path & cgroup);
 
 }
 

From e6b71f84a0a766429fdceaf188ea0167e36a20d9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Nov 2022 16:59:36 +0100
Subject: [PATCH 081/112] Use cgroup.kill to quickly kill cgroups

---
 src/libstore/cgroup.cc | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc
index 2a485f0f9..f693d77be 100644
--- a/src/libstore/cgroup.cc
+++ b/src/libstore/cgroup.cc
@@ -35,9 +35,19 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats)
 {
     if (!pathExists(cgroup)) return {};
 
-    if (!pathExists(cgroup + "/cgroup.procs"))
+    auto procsFile = cgroup + "/cgroup.procs";
+
+    if (!pathExists(procsFile))
         throw Error("'%s' is not a cgroup", cgroup);
 
+    /* Use the fast way to kill every process in a cgroup, if
+       available. */
+    auto killFile = cgroup + "/cgroup.kill";
+    if (pathExists(killFile))
+        writeFile(killFile, "1");
+
+    /* Otherwise, manually kill every process in the subcgroups and
+       this cgroup. */
     for (auto & entry : readDirectory(cgroup)) {
         if (entry.type != DT_DIR) continue;
         destroyCgroup(cgroup + "/" + entry.name, false);
@@ -48,7 +58,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats)
     std::unordered_set<pid_t> pidsShown;
 
     while (true) {
-        auto pids = tokenizeString<std::vector<std::string>>(readFile(cgroup + "/cgroup.procs"));
+        auto pids = tokenizeString<std::vector<std::string>>(readFile(procsFile));
 
         if (pids.empty()) break;
 

From f538ee434285304cb61cf10bf13127f13bfced1b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 09:38:08 +0100
Subject: [PATCH 082/112] Rename derivedPathsWithHintsToJSON ->
 builtPathsToJSON

---
 src/libstore/derived-path.cc | 2 +-
 src/libstore/derived-path.hh | 2 +-
 src/nix/build.cc             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 44587ae78..7fe9b9648 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -64,7 +64,7 @@ nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) {
     return res;
 }
 
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store)
+nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store)
 { return stuffToJSON<BuiltPath>(buildables, store); }
 nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
 { return stuffToJSON<DerivedPath>(paths, store); }
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index 24a0ae773..b2d0956b8 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -125,7 +125,7 @@ struct BuiltPath : _BuiltPathRaw {
 typedef std::vector<DerivedPath> DerivedPaths;
 typedef std::vector<BuiltPath> BuiltPaths;
 
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store);
+nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store);
 nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store);
 
 }
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 9c648d28e..2b91f8e0a 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -78,7 +78,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
             Realise::Outputs,
             installables, buildMode);
 
-        if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump());
+        if (json) logger->cout("%s", builtPathsToJSON(buildables, store).dump());
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())

From 300753d594fd7cd818d08f9c7a18a9ebc305bd95 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 10:49:01 +0100
Subject: [PATCH 083/112] nix build --json: Include build statistics

Example:

  # nix build -L --extra-experimental-features cgroups --impure --expr 'with import <nixpkgs> {}; runCommand "foo" {} "dd if=/dev/urandom bs=1M count=1024 | md5sum; mkdir $out"' --json
  [
    {
      "cpuSystem": 1.911431,
      "cpuUser": 1.214249,
      "drvPath": "/nix/store/xzdqz67xba18hljhycp0hwfigzrs2z69-foo.drv",
      "outputs": {
        "out": "/nix/store/rh9mc9l2gkpq8kn2sgzndr6ll7ffjh6l-foo"
      },
      "startTime": 1669024076,
      "stopTime": 1669024079
    }
  ]
---
 src/libcmd/installables.cc   | 29 ++++++++++++++-----------
 src/libcmd/installables.hh   | 11 ++++++++--
 src/libstore/derived-path.cc | 23 ++++----------------
 src/libstore/derived-path.hh |  3 ---
 src/nix/app.cc               | 11 +++++++---
 src/nix/build.cc             | 42 ++++++++++++++++++++++++++++++++----
 src/nix/profile.cc           |  4 ++--
 7 files changed, 77 insertions(+), 46 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 5945a2578..e036b8836 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -840,20 +840,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths Installable::build(
+std::vector<BuiltPathWithResult> Installable::build(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     const std::vector<std::shared_ptr<Installable>> & installables,
     BuildMode bMode)
 {
-    BuiltPaths res;
-    for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
-        res.push_back(builtPath);
+    std::vector<BuiltPathWithResult> res;
+    for (auto & [_, builtPathWithResult] : build2(evalStore, store, mode, installables, bMode))
+        res.push_back(builtPathWithResult);
     return res;
 }
 
-std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
+std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -873,7 +873,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
         }
     }
 
-    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
+    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> res;
 
     switch (mode) {
 
@@ -914,10 +914,10 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                                     output, *drvOutput->second);
                             }
                         }
-                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                        res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
-                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                        res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}});
                     },
                 }, path.raw());
             }
@@ -939,10 +939,10 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                         std::map<std::string, StorePath> outputs;
                         for (auto & path : buildResult.builtOutputs)
                             outputs.emplace(path.first.outputName, path.second.outPath);
-                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                        res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
-                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                        res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}});
                     },
                 }, buildResult.path.raw());
             }
@@ -965,9 +965,12 @@ BuiltPaths Installable::toBuiltPaths(
     OperateOn operateOn,
     const std::vector<std::shared_ptr<Installable>> & installables)
 {
-    if (operateOn == OperateOn::Output)
-        return Installable::build(evalStore, store, mode, installables);
-    else {
+    if (operateOn == OperateOn::Output) {
+        BuiltPaths res;
+        for (auto & p : Installable::build(evalStore, store, mode, installables))
+            res.push_back(p.path);
+        return res;
+    } else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index 948f78919..02ea351d3 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -7,6 +7,7 @@
 #include "eval.hh"
 #include "store-api.hh"
 #include "flake/flake.hh"
+#include "build-result.hh"
 
 #include <optional>
 
@@ -51,6 +52,12 @@ enum class OperateOn {
     Derivation
 };
 
+struct BuiltPathWithResult
+{
+    BuiltPath path;
+    std::optional<BuildResult> result;
+};
+
 struct Installable
 {
     virtual ~Installable() { }
@@ -91,14 +98,14 @@ struct Installable
         return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
     }
 
-    static BuiltPaths build(
+    static std::vector<BuiltPathWithResult> build(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
         const std::vector<std::shared_ptr<Installable>> & installables,
         BuildMode bMode = bmNormal);
 
-    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
+    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> build2(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 7fe9b9648..88b59f615 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -53,28 +53,13 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
-template<typename T>
-nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) {
-    auto res = nlohmann::json::array();
-    for (const T & t : ts) {
-        std::visit([&res, store](const auto & t) {
-            res.push_back(t.toJSON(store));
-        }, t.raw());
-    }
-    return res;
-}
-
-nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store)
-{ return stuffToJSON<BuiltPath>(buildables, store); }
-nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
-{ return stuffToJSON<DerivedPath>(paths, store); }
-
-
-std::string DerivedPath::Opaque::to_string(const Store & store) const {
+std::string DerivedPath::Opaque::to_string(const Store & store) const
+{
     return store.printStorePath(path);
 }
 
-std::string DerivedPath::Built::to_string(const Store & store) const {
+std::string DerivedPath::Built::to_string(const Store & store) const
+{
     return store.printStorePath(drvPath)
         + "!"
         + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index b2d0956b8..878696136 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -125,7 +125,4 @@ struct BuiltPath : _BuiltPathRaw {
 typedef std::vector<DerivedPath> DerivedPaths;
 typedef std::vector<BuiltPath> BuiltPaths;
 
-nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store);
-nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store);
-
 }
diff --git a/src/nix/app.cc b/src/nix/app.cc
index 48de8fb82..5658f2a52 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -37,11 +37,13 @@ struct InstallableDerivedPath : Installable
  * Return the rewrites that are needed to resolve a string whose context is
  * included in `dependencies`.
  */
-StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies)
+StringPairs resolveRewrites(
+    Store & store,
+    const std::vector<BuiltPathWithResult> & dependencies)
 {
     StringPairs res;
     for (auto & dep : dependencies)
-        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep))
+        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
             for (auto & [ outputName, outputPath ] : drvDep->outputs)
                 res.emplace(
                     downstreamPlaceholder(store, drvDep->drvPath, outputName),
@@ -53,7 +55,10 @@ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies)
 /**
  * Resolve the given string assuming the given context.
  */
-std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies)
+std::string resolveString(
+    Store & store,
+    const std::string & toResolve,
+    const std::vector<BuiltPathWithResult> & dependencies)
 {
     auto rewrites = resolveRewrites(store, dependencies);
     return rewriteStrings(toResolve, rewrites);
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 2b91f8e0a..94b169167 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -10,6 +10,37 @@
 
 using namespace nix;
 
+nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
+{
+    auto res = nlohmann::json::array();
+    for (auto & t : paths) {
+        std::visit([&res, store](const auto & t) {
+            res.push_back(t.toJSON(store));
+        }, t.raw());
+    }
+    return res;
+}
+
+nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, ref<Store> store)
+{
+    auto res = nlohmann::json::array();
+    for (auto & b : buildables) {
+        std::visit([&](const auto & t) {
+            auto j = t.toJSON(store);
+            if (b.result) {
+                j["startTime"] = b.result->startTime;
+                j["stopTime"] = b.result->stopTime;
+                if (b.result->cpuUser)
+                    j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000;
+                if (b.result->cpuSystem)
+                    j["cpuSystem"] = ((double) b.result->cpuSystem->count()) / 1000000;
+            }
+            res.push_back(j);
+        }, b.path.raw());
+    }
+    return res;
+}
+
 struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 {
     Path outLink = "result";
@@ -78,7 +109,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
             Realise::Outputs,
             installables, buildMode);
 
-        if (json) logger->cout("%s", builtPathsToJSON(buildables, store).dump());
+        if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump());
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
@@ -98,7 +129,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                                 store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
-                    }, buildable.raw());
+                    }, buildable.path.raw());
                 }
 
         if (printOutputPaths) {
@@ -113,11 +144,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                             std::cout << store->printStorePath(output.second) << std::endl;
                         }
                     },
-                }, buildable.raw());
+                }, buildable.path.raw());
             }
         }
 
-        updateProfile(buildables);
+        BuiltPaths buildables2;
+        for (auto & b : buildables)
+            buildables2.push_back(b.path);
+        updateProfile(buildables2);
     }
 };
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 3814e7d5a..11910523d 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -253,11 +253,11 @@ struct ProfileManifest
 
 static std::map<Installable *, BuiltPaths>
 builtPathsPerInstallable(
-    const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths)
+    const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths)
 {
     std::map<Installable *, BuiltPaths> res;
     for (auto & [installable, builtPath] : builtPaths)
-        res[installable.get()].push_back(builtPath);
+        res[installable.get()].push_back(builtPath.path);
     return res;
 }
 

From ec45f4b82eaef8da04f4b828b2b06a77aa3f986f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 11:12:45 +0100
Subject: [PATCH 084/112] Fix indentation

---
 src/libstore/local-store.cc | 50 ++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 04223d860..b67668e52 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -897,48 +897,48 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
 
 std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & state, const StorePath & path)
 {
-            /* Get the path info. */
+    /* Get the path info. */
     auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path)));
 
-            if (!useQueryPathInfo.next())
-                return std::shared_ptr<ValidPathInfo>();
+    if (!useQueryPathInfo.next())
+        return std::shared_ptr<ValidPathInfo>();
 
-            auto id = useQueryPathInfo.getInt(0);
+    auto id = useQueryPathInfo.getInt(0);
 
-            auto narHash = Hash::dummy;
-            try {
-                narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
-            } catch (BadHash & e) {
-                throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what());
-            }
+    auto narHash = Hash::dummy;
+    try {
+        narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
+    } catch (BadHash & e) {
+        throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what());
+    }
 
-            auto info = std::make_shared<ValidPathInfo>(path, narHash);
+    auto info = std::make_shared<ValidPathInfo>(path, narHash);
 
-            info->id = id;
+    info->id = id;
 
-            info->registrationTime = useQueryPathInfo.getInt(2);
+    info->registrationTime = useQueryPathInfo.getInt(2);
 
     auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3);
-            if (s) info->deriver = parseStorePath(s);
+    if (s) info->deriver = parseStorePath(s);
 
-            /* Note that narSize = NULL yields 0. */
-            info->narSize = useQueryPathInfo.getInt(4);
+    /* Note that narSize = NULL yields 0. */
+    info->narSize = useQueryPathInfo.getInt(4);
 
-            info->ultimate = useQueryPathInfo.getInt(5) == 1;
+    info->ultimate = useQueryPathInfo.getInt(5) == 1;
 
     s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6);
-            if (s) info->sigs = tokenizeString<StringSet>(s, " ");
+    if (s) info->sigs = tokenizeString<StringSet>(s, " ");
 
     s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
-            if (s) info->ca = parseContentAddressOpt(s);
+    if (s) info->ca = parseContentAddressOpt(s);
 
-            /* Get the references. */
+    /* Get the references. */
     auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
 
-            while (useQueryReferences.next())
-                info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
+    while (useQueryReferences.next())
+        info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
 
-            return info;
+    return info;
 }
 
 
@@ -1041,9 +1041,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
     auto path = path_;
     auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
         auto state(_state.lock());
-    std::map<std::string, std::optional<StorePath>> outputs;
+        std::map<std::string, std::optional<StorePath>> outputs;
         uint64_t drvId;
-            drvId = queryValidPathId(*state, path);
+        drvId = queryValidPathId(*state, path);
         auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
         while (use.next())
             outputs.insert_or_assign(

From 82d5cf2a76ec009fd94a925c22a5e099a0b7321b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 11:45:41 +0100
Subject: [PATCH 085/112] Fix macOS build

---
 src/libstore/build/local-derivation-goal.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index f273ebe8a..34f8ab5f1 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -149,11 +149,15 @@ void LocalDerivationGoal::killChild()
 void LocalDerivationGoal::killSandbox(bool getStats)
 {
     if (cgroup) {
+        #if __linux__
         auto stats = destroyCgroup(*cgroup);
         if (getStats) {
             buildResult.cpuUser = stats.cpuUser;
             buildResult.cpuSystem = stats.cpuSystem;
         }
+        #else
+        abort();
+        #endif
     }
 
     else if (buildUser) {

From e7a5b76844a649645e51a60dd18fd383d14d8755 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 09:38:08 +0100
Subject: [PATCH 086/112] Rename derivedPathsWithHintsToJSON ->
 builtPathsToJSON

---
 src/libstore/derived-path.cc | 2 +-
 src/libstore/derived-path.hh | 2 +-
 src/nix/build.cc             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 44587ae78..7fe9b9648 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -64,7 +64,7 @@ nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) {
     return res;
 }
 
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store)
+nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store)
 { return stuffToJSON<BuiltPath>(buildables, store); }
 nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
 { return stuffToJSON<DerivedPath>(paths, store); }
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index 24a0ae773..b2d0956b8 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -125,7 +125,7 @@ struct BuiltPath : _BuiltPathRaw {
 typedef std::vector<DerivedPath> DerivedPaths;
 typedef std::vector<BuiltPath> BuiltPaths;
 
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store);
+nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store);
 nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store);
 
 }
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 9c648d28e..2b91f8e0a 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -78,7 +78,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
             Realise::Outputs,
             installables, buildMode);
 
-        if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump());
+        if (json) logger->cout("%s", builtPathsToJSON(buildables, store).dump());
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())

From f0baa5c1283359a413ca3a254527587c86b2f097 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 10:49:01 +0100
Subject: [PATCH 087/112] nix build --json: Include build statistics

Example:

  # nix build -L --extra-experimental-features cgroups --impure --expr 'with import <nixpkgs> {}; runCommand "foo" {} "dd if=/dev/urandom bs=1M count=1024 | md5sum; mkdir $out"' --json
  [
    {
      "cpuSystem": 1.911431,
      "cpuUser": 1.214249,
      "drvPath": "/nix/store/xzdqz67xba18hljhycp0hwfigzrs2z69-foo.drv",
      "outputs": {
        "out": "/nix/store/rh9mc9l2gkpq8kn2sgzndr6ll7ffjh6l-foo"
      },
      "startTime": 1669024076,
      "stopTime": 1669024079
    }
  ]
---
 src/libcmd/installables.cc   | 29 +++++++++++++++------------
 src/libcmd/installables.hh   | 11 +++++++++--
 src/libstore/derived-path.cc | 23 ++++------------------
 src/libstore/derived-path.hh |  3 ---
 src/nix/app.cc               | 11 ++++++++---
 src/nix/build.cc             | 38 ++++++++++++++++++++++++++++++++----
 src/nix/profile.cc           |  4 ++--
 7 files changed, 73 insertions(+), 46 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index f63b9eeae..d6e62e775 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -844,20 +844,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths Installable::build(
+std::vector<BuiltPathWithResult> Installable::build(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     const std::vector<std::shared_ptr<Installable>> & installables,
     BuildMode bMode)
 {
-    BuiltPaths res;
-    for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
-        res.push_back(builtPath);
+    std::vector<BuiltPathWithResult> res;
+    for (auto & [_, builtPathWithResult] : build2(evalStore, store, mode, installables, bMode))
+        res.push_back(builtPathWithResult);
     return res;
 }
 
-std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
+std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -877,7 +877,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
         }
     }
 
-    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
+    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> res;
 
     switch (mode) {
 
@@ -918,10 +918,10 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                                     output, *drvOutput->second);
                             }
                         }
-                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                        res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
-                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                        res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}});
                     },
                 }, path.raw());
             }
@@ -943,10 +943,10 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                         std::map<std::string, StorePath> outputs;
                         for (auto & path : buildResult.builtOutputs)
                             outputs.emplace(path.first.outputName, path.second.outPath);
-                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                        res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
-                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                        res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}});
                     },
                 }, buildResult.path.raw());
             }
@@ -969,9 +969,12 @@ BuiltPaths Installable::toBuiltPaths(
     OperateOn operateOn,
     const std::vector<std::shared_ptr<Installable>> & installables)
 {
-    if (operateOn == OperateOn::Output)
-        return Installable::build(evalStore, store, mode, installables);
-    else {
+    if (operateOn == OperateOn::Output) {
+        BuiltPaths res;
+        for (auto & p : Installable::build(evalStore, store, mode, installables))
+            res.push_back(p.path);
+        return res;
+    } else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index 948f78919..02ea351d3 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -7,6 +7,7 @@
 #include "eval.hh"
 #include "store-api.hh"
 #include "flake/flake.hh"
+#include "build-result.hh"
 
 #include <optional>
 
@@ -51,6 +52,12 @@ enum class OperateOn {
     Derivation
 };
 
+struct BuiltPathWithResult
+{
+    BuiltPath path;
+    std::optional<BuildResult> result;
+};
+
 struct Installable
 {
     virtual ~Installable() { }
@@ -91,14 +98,14 @@ struct Installable
         return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
     }
 
-    static BuiltPaths build(
+    static std::vector<BuiltPathWithResult> build(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
         const std::vector<std::shared_ptr<Installable>> & installables,
         BuildMode bMode = bmNormal);
 
-    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
+    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> build2(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 7fe9b9648..88b59f615 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -53,28 +53,13 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
-template<typename T>
-nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) {
-    auto res = nlohmann::json::array();
-    for (const T & t : ts) {
-        std::visit([&res, store](const auto & t) {
-            res.push_back(t.toJSON(store));
-        }, t.raw());
-    }
-    return res;
-}
-
-nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store)
-{ return stuffToJSON<BuiltPath>(buildables, store); }
-nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
-{ return stuffToJSON<DerivedPath>(paths, store); }
-
-
-std::string DerivedPath::Opaque::to_string(const Store & store) const {
+std::string DerivedPath::Opaque::to_string(const Store & store) const
+{
     return store.printStorePath(path);
 }
 
-std::string DerivedPath::Built::to_string(const Store & store) const {
+std::string DerivedPath::Built::to_string(const Store & store) const
+{
     return store.printStorePath(drvPath)
         + "!"
         + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index b2d0956b8..878696136 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -125,7 +125,4 @@ struct BuiltPath : _BuiltPathRaw {
 typedef std::vector<DerivedPath> DerivedPaths;
 typedef std::vector<BuiltPath> BuiltPaths;
 
-nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref<Store> store);
-nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store);
-
 }
diff --git a/src/nix/app.cc b/src/nix/app.cc
index 48de8fb82..5658f2a52 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -37,11 +37,13 @@ struct InstallableDerivedPath : Installable
  * Return the rewrites that are needed to resolve a string whose context is
  * included in `dependencies`.
  */
-StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies)
+StringPairs resolveRewrites(
+    Store & store,
+    const std::vector<BuiltPathWithResult> & dependencies)
 {
     StringPairs res;
     for (auto & dep : dependencies)
-        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep))
+        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
             for (auto & [ outputName, outputPath ] : drvDep->outputs)
                 res.emplace(
                     downstreamPlaceholder(store, drvDep->drvPath, outputName),
@@ -53,7 +55,10 @@ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies)
 /**
  * Resolve the given string assuming the given context.
  */
-std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies)
+std::string resolveString(
+    Store & store,
+    const std::string & toResolve,
+    const std::vector<BuiltPathWithResult> & dependencies)
 {
     auto rewrites = resolveRewrites(store, dependencies);
     return rewriteStrings(toResolve, rewrites);
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 2b91f8e0a..85b1efc33 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -10,6 +10,33 @@
 
 using namespace nix;
 
+nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
+{
+    auto res = nlohmann::json::array();
+    for (auto & t : paths) {
+        std::visit([&res, store](const auto & t) {
+            res.push_back(t.toJSON(store));
+        }, t.raw());
+    }
+    return res;
+}
+
+nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, ref<Store> store)
+{
+    auto res = nlohmann::json::array();
+    for (auto & b : buildables) {
+        std::visit([&](const auto & t) {
+            auto j = t.toJSON(store);
+            if (b.result) {
+                j["startTime"] = b.result->startTime;
+                j["stopTime"] = b.result->stopTime;
+            }
+            res.push_back(j);
+        }, b.path.raw());
+    }
+    return res;
+}
+
 struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 {
     Path outLink = "result";
@@ -78,7 +105,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
             Realise::Outputs,
             installables, buildMode);
 
-        if (json) logger->cout("%s", builtPathsToJSON(buildables, store).dump());
+        if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump());
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
@@ -98,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                                 store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
-                    }, buildable.raw());
+                    }, buildable.path.raw());
                 }
 
         if (printOutputPaths) {
@@ -113,11 +140,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                             std::cout << store->printStorePath(output.second) << std::endl;
                         }
                     },
-                }, buildable.raw());
+                }, buildable.path.raw());
             }
         }
 
-        updateProfile(buildables);
+        BuiltPaths buildables2;
+        for (auto & b : buildables)
+            buildables2.push_back(b.path);
+        updateProfile(buildables2);
     }
 };
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 3814e7d5a..11910523d 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -253,11 +253,11 @@ struct ProfileManifest
 
 static std::map<Installable *, BuiltPaths>
 builtPathsPerInstallable(
-    const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths)
+    const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths)
 {
     std::map<Installable *, BuiltPaths> res;
     for (auto & [installable, builtPath] : builtPaths)
-        res[installable.get()].push_back(builtPath);
+        res[installable.get()].push_back(builtPath.path);
     return res;
 }
 

From 9d17ce07e872e88057480744414e0d1ef4fd5fa8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 12:55:49 +0100
Subject: [PATCH 088/112] AutoUserLock: If sandboxing is disabled, use the
 build users group

We have to use a gid that has write access to the Nix store.
---
 src/libstore/build/local-derivation-goal.cc | 60 ++++++++++-----------
 src/libstore/lock.cc                        | 22 ++++----
 src/libstore/lock.hh                        |  2 +-
 3 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 34f8ab5f1..b7084384a 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -177,9 +177,38 @@ void LocalDerivationGoal::tryLocalBuild() {
         return;
     }
 
+    /* Are we doing a chroot build? */
+    {
+        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
+        if (settings.sandboxMode == smEnabled) {
+            if (noChroot)
+                throw Error("derivation '%s' has '__noChroot' set, "
+                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
+#if __APPLE__
+            if (additionalSandboxProfile != "")
+                throw Error("derivation '%s' specifies a sandbox profile, "
+                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
+#endif
+            useChroot = true;
+        }
+        else if (settings.sandboxMode == smDisabled)
+            useChroot = false;
+        else if (settings.sandboxMode == smRelaxed)
+            useChroot = derivationType.isSandboxed() && !noChroot;
+    }
+
+    auto & localStore = getLocalStore();
+    if (localStore.storeDir != localStore.realStoreDir.get()) {
+        #if __linux__
+            useChroot = true;
+        #else
+            throw Error("building using a diverted store is not supported on this platform");
+        #endif
+    }
+
     if (useBuildUsers()) {
         if (!buildUser)
-            buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1);
+            buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot);
 
         if (!buildUser) {
             if (!actLock)
@@ -433,35 +462,6 @@ void LocalDerivationGoal::startBuilder()
     additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
 #endif
 
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = derivationType.isSandboxed() && !noChroot;
-    }
-
-    auto & localStore = getLocalStore();
-    if (localStore.storeDir != localStore.realStoreDir.get()) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
     /* Create a temporary directory where the build will take
        place. */
     tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index 4fad3bfd2..3b93979a8 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -109,22 +109,18 @@ struct AutoUserLock : UserLock
 {
     AutoCloseFD fdUserLock;
     uid_t firstUid = 0;
+    gid_t firstGid = 0;
     uid_t nrIds = 1;
 
     uid_t getUID() override { assert(firstUid); return firstUid; }
 
     gid_t getUIDCount() override { return nrIds; }
 
-    gid_t getGID() override
-    {
-        // We use the same GID ranges as for the UIDs.
-        assert(firstUid);
-        return firstUid;
-    }
+    gid_t getGID() override { assert(firstGid); return firstGid; }
 
     std::vector<gid_t> getSupplementaryGIDs() override { return {}; }
 
-    static std::unique_ptr<UserLock> acquire(uid_t nrIds)
+    static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useChroot)
     {
         settings.requireExperimentalFeature(Xp::AutoAllocateUids);
         assert(settings.startId > 0);
@@ -154,6 +150,14 @@ struct AutoUserLock : UserLock
                 auto lock = std::make_unique<AutoUserLock>();
                 lock->fdUserLock = std::move(fd);
                 lock->firstUid = settings.startId + i * maxIdsPerBuild;
+                if (useChroot)
+                    lock->firstGid = lock->firstUid;
+                else {
+                    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
+                    if (!gr)
+                        throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
+                    lock->firstGid = gr->gr_gid;
+                }
                 lock->nrIds = nrIds;
                 return lock;
             }
@@ -163,10 +167,10 @@ struct AutoUserLock : UserLock
     }
 };
 
-std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds)
+std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot)
 {
     if (settings.autoAllocateUids)
-        return AutoUserLock::acquire(nrIds);
+        return AutoUserLock::acquire(nrIds, useChroot);
     else
         return SimpleUserLock::acquire();
 }
diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh
index e7ceefab8..49ad86de7 100644
--- a/src/libstore/lock.hh
+++ b/src/libstore/lock.hh
@@ -31,7 +31,7 @@ struct UserLock
 
 /* Acquire a user lock for a UID range of size `nrIds`. Note that this
    may return nullptr if no user is available. */
-std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds);
+std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot);
 
 bool useBuildUsers();
 

From c776dfbb35e961ac3f011ab8665dfc85ab067ef8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Nov 2022 18:46:55 +0100
Subject: [PATCH 089/112] Use hex for startId

Co-authored-by: Linus Heckemann <git@sphalerite.org>
---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 88fe72202..653d108aa 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -288,7 +288,7 @@ public:
 
     Setting<uint32_t> startId{this,
         #if __linux__
-        872415232,
+        0x34000000,
         #else
         56930,
         #endif

From b37c2d84b67635fc928ed174166f04d6f4d30c6b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Nov 2022 09:02:17 +0100
Subject: [PATCH 090/112] Always call setgroups()

We shouldn't skip this if the supplementary group list is empty,
because then the sandbox won't drop the supplementary groups of the
parent (like "root").
---
 src/libstore/build/local-derivation-goal.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index b7084384a..232440f74 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1988,9 +1988,8 @@ void LocalDerivationGoal::runChild()
         if (setUser && buildUser) {
             /* Preserve supplementary groups of the build user, to allow
                admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
+            auto gids = buildUser->getSupplementaryGIDs();
+            if (setgroups(gids.size(), gids.data()) == -1)
                 throw SysError("cannot set supplementary groups of build user");
 
             if (setgid(buildUser->getGID()) == -1 ||

From 3d23b9d0324ff415af9e5f35568aca98c04a90cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Nov 2022 09:03:30 +0100
Subject: [PATCH 091/112] SimpleUserLock::getSupplementaryGIDs(): Filter out
 main gid

This avoids having the user's gid in the supplementary group list as
well.
---
 src/libstore/lock.cc | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index 3b93979a8..7459d837d 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -71,21 +71,22 @@ struct SimpleUserLock : UserLock
                    user.  This is usually either empty or contains a
                    group such as "kvm".  */
                 int ngroups = 32; // arbitrary initial guess
-                lock->supplementaryGIDs.resize(ngroups);
+                std::vector<gid_t> gids;
+                gids.resize(ngroups);
 
                 int err = getgrouplist(
                     pw->pw_name, pw->pw_gid,
-                    lock->supplementaryGIDs.data(),
+                    gids.data(),
                     &ngroups);
 
                 /* Our initial size of 32 wasn't sufficient, the
                    correct size has been stored in ngroups, so we try
                    again. */
                 if (err == -1) {
-                    lock->supplementaryGIDs.resize(ngroups);
+                    gids.resize(ngroups);
                     err = getgrouplist(
                         pw->pw_name, pw->pw_gid,
-                        lock->supplementaryGIDs.data(),
+                        gids.data(),
                         &ngroups);
                 }
 
@@ -94,7 +95,9 @@ struct SimpleUserLock : UserLock
                     throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name);
 
                 // Finally, trim back the GID list to its real size.
-                lock->supplementaryGIDs.resize(ngroups);
+                for (auto i = 0; i < ngroups; i++)
+                    if (gids[i] != lock->gid)
+                        lock->supplementaryGIDs.push_back(gids[i]);
                 #endif
 
                 return lock;

From 52f0c809173fe977ac0b46a8506d0c9af208a197 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 22 Nov 2022 10:36:20 +0100
Subject: [PATCH 092/112] fix error in language overview

it is not possible to antiquote numbers.
---
 doc/manual/src/language/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md
index f9e9b9781..db34fde75 100644
--- a/doc/manual/src/language/index.md
+++ b/doc/manual/src/language/index.md
@@ -93,7 +93,7 @@ This is an incomplete overview of language features, by example.
 
    `"hello ${ { a = "world" }.a }"`
 
-   `"1 2 ${3}"`
+   `"1 2 ${toString 3}"`
 
    `"${pkgs.bash}/bin/sh"`
 

From 989fc8a8b9cf98addbef85bf909be7b00b0462db Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Nov 2022 15:24:50 +0100
Subject: [PATCH 093/112] Add release notes

---
 doc/manual/src/release-notes/rl-next.md | 40 +++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 2069e4578..47181fd39 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -10,3 +10,43 @@
   This avoids a lot of spurious errors where some benign strings end-up having
   a context just because they are read from a store path
   ([#7260](https://github.com/NixOS/nix/pull/7260)).
+
+* Nix can now automatically pick UIDs for builds, removing the need to
+  create `nixbld*` user accounts. these UIDs are allocated starting at
+  872415232 on Linux and 56930 on macOS.
+
+  This is an experimental feature. To enable it, add the following to
+  `nix.conf`:
+
+  ```
+  extra-experimental-features = auto-allocate-uids
+  auto-allocate-uids = true
+  ```
+
+* On Linux, Nix can now run builds in a user namespace where the build
+  runs as root (UID 0) and has 65,536 UIDs available. This is
+  primarily useful for running containers such as `systemd-nspawn`
+  inside a Nix build.
+
+  A build can enable this by requiring the `uid-range` system feature,
+  i.e. by setting the derivation attribute
+
+  ```
+  requiredSystemFeatures = [ "uid-range" ];
+  ```
+
+  The `uid-range` system feature requires the `auto-allocate-uids`
+  setting to be enabled (see above).
+
+* On Linux, Nix has experimental support for running builds inside a
+  cgroup. It can be enabled by adding
+
+  ```
+  extra-experimental-features = cgroups
+  ```
+
+  to `nix.conf`. It is also automatically enabled for builds that
+  require the `uid-range` system feature.
+
+* `nix build --json` now prints some statistics about top-level
+  derivations, such as CPU statistics when cgroups are enabled.

From 2aa3f2e81020c1c780be6329e1133068779c8f08 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Nov 2022 17:07:59 +0100
Subject: [PATCH 094/112] Include UID in hex

---
 doc/manual/src/release-notes/rl-next.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 47181fd39..4c91002fb 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -12,8 +12,8 @@
   ([#7260](https://github.com/NixOS/nix/pull/7260)).
 
 * Nix can now automatically pick UIDs for builds, removing the need to
-  create `nixbld*` user accounts. these UIDs are allocated starting at
-  872415232 on Linux and 56930 on macOS.
+  create `nixbld*` user accounts. These UIDs are allocated starting at
+  872415232 (0x34000000) on Linux and 56930 on macOS.
 
   This is an experimental feature. To enable it, add the following to
   `nix.conf`:

From b13fd4c58e81b2b2b0d72caa5ce80de861622610 Mon Sep 17 00:00:00 2001
From: Taeer Bar-Yam <taeer.bar-yam@tweag.io>
Date: Wed, 23 Nov 2022 11:39:50 -0500
Subject: [PATCH 095/112] Fix why-depends for CA derivations

why-depends assumed that we knew the output path of the second argument.
For CA derivations, we might not know until it's built. One way to solve
this would be to build the second installable to get the output path.

In this case we don't need to, though. If the first installable (A)
depends on the second (B), then getting the store path of A will
necessitate having the store path B. The contrapositive is, if the store
path of B is not known (i.e. it's a CA derivation which hasn't been
built), then A does not depend on B.
---
 src/nix/why-depends.cc | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 1d9ab28ba..285e36722 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -84,19 +84,35 @@ struct CmdWhyDepends : SourceExprCommand
         auto package = parseInstallable(store, _package);
         auto packagePath = Installable::toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package);
         auto dependency = parseInstallable(store, _dependency);
-        auto dependencyPath = Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency);
-        auto dependencyPathHash = dependencyPath.hashPart();
+        auto derivedDependency = dependency->toDerivedPath();
+        auto optDependencyPath = std::visit(overloaded {
+            [](const DerivedPath::Opaque & nodrv) -> std::optional<StorePath> {
+                return { nodrv.path };
+            },
+            [&](const DerivedPath::Built & hasdrv) -> std::optional<StorePath> {
+                if (hasdrv.outputs.size() != 1) {
+                    throw Error("argument '%s' should evaluate to one store path", dependency->what());
+                }
+                auto outputMap = store->queryPartialDerivationOutputMap(hasdrv.drvPath);
+                auto maybePath = outputMap.find(*hasdrv.outputs.begin());
+                if (maybePath == outputMap.end()) {
+                    throw Error("unexpected end of iterator");
+                }
+                return maybePath->second;
+            },
+        }, derivedDependency.raw());
 
         StorePathSet closure;
         store->computeFSClosure({packagePath}, closure, false, false);
 
-        if (!closure.count(dependencyPath)) {
-            printError("'%s' does not depend on '%s'",
-                store->printStorePath(packagePath),
-                store->printStorePath(dependencyPath));
+        if (!optDependencyPath.has_value() || !closure.count(*optDependencyPath)) {
+            printError("'%s' does not depend on '%s'", package->what(), dependency->what());
             return;
         }
 
+        auto dependencyPath = *optDependencyPath;
+        auto dependencyPathHash = dependencyPath.hashPart();
+
         stopProgressBar(); // FIXME
 
         auto accessor = store->getFSAccessor();

From bd8571a5c3724ba5917564a5243af173966515c5 Mon Sep 17 00:00:00 2001
From: Taeer Bar-Yam <taeer.bar-yam@tweag.io>
Date: Wed, 23 Nov 2022 12:06:47 -0500
Subject: [PATCH 096/112] add explanation and test

---
 src/nix/why-depends.cc  | 11 +++++++++++
 tests/ca/why-depends.sh |  5 +++++
 2 files changed, 16 insertions(+)
 create mode 100644 tests/ca/why-depends.sh

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 285e36722..723017497 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -83,6 +83,17 @@ struct CmdWhyDepends : SourceExprCommand
     {
         auto package = parseInstallable(store, _package);
         auto packagePath = Installable::toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package);
+
+        /* We don't need to build `dependency`. We try to get the store
+         * path if it's already known, and if not, then it's not a dependency.
+         *
+         * Why? If `package` does depends on `dependency`, then getting the
+         * store path of `package` above necessitated having the store path
+         * of `dependency`. The contrapositive is, if the store path of
+         * `dependency` is not already known at this point (i.e. it's a CA
+         * derivation which hasn't been built), then `package` did not need it
+         * to build.
+         */
         auto dependency = parseInstallable(store, _dependency);
         auto derivedDependency = dependency->toDerivedPath();
         auto optDependencyPath = std::visit(overloaded {
diff --git a/tests/ca/why-depends.sh b/tests/ca/why-depends.sh
new file mode 100644
index 000000000..0c079f63b
--- /dev/null
+++ b/tests/ca/why-depends.sh
@@ -0,0 +1,5 @@
+source common.sh
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+
+cd .. && source why-depends.sh

From d6318e1638ed17fef701ada19ec7fde4b54d3bcf Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 24 Nov 2022 13:47:08 +0100
Subject: [PATCH 097/112] refactor rendering documentation of builtins

as in [1], make the document structure visible, like in a template

[1]: 4655563470b59e0ef50a33af003058c2b54db778
---
 doc/manual/generate-builtins.nix | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
index 6c8b88da2..115bb3f94 100644
--- a/doc/manual/generate-builtins.nix
+++ b/doc/manual/generate-builtins.nix
@@ -1,16 +1,20 @@
-with builtins;
-with import ./utils.nix;
+builtinsDump:
+let
+  showBuiltin = name:
+    let
+      inherit (builtinsDump.${name}) doc args;
+    in
+    ''
+      <dt id="builtins-${name}">
+        <a href="#builtins-${name}"><code>${name} ${listArgs args}</code></a>
+      </dt>
+      <dd>
 
-builtins:
+        ${doc}
+
+      </dd>
+    '';
+  listArgs = args: builtins.concatStringsSep " " (map (s: "<var>${s}</var>") args);
+in
+with builtins; concatStringsSep "\n" (map showBuiltin (attrNames builtinsDump))
 
-concatStrings (map
-  (name:
-    let builtin = builtins.${name}; in
-    "<dt id=\"builtins-${name}\"><a href=\"#builtins-${name}\"><code>${name} "
-    + concatStringsSep " " (map (s: "<var>${s}</var>") builtin.args)
-    + "</code></a></dt>"
-    + "<dd>\n\n"
-    + builtin.doc
-    + "\n\n</dd>"
-  )
-  (attrNames builtins))

From 0b4c4d74344613a15791e61179a441d80803d67a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Nov 2022 22:30:56 +0100
Subject: [PATCH 098/112] Don't use GC_STRNDUP

It calls strlen() on the input (rather than simply copying at most
`size` bytes), which can fail if the input is not zero-terminated and
is inefficient in any case.

Fixes #7347.
---
 src/libexpr/eval.cc | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 563f24e48..6ba44cc1d 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -69,15 +69,11 @@ static char * dupString(const char * s)
 // empty string.
 static const char * makeImmutableStringWithLen(const char * s, size_t size)
 {
-    char * t;
     if (size == 0)
         return "";
-#if HAVE_BOEHMGC
-    t = GC_STRNDUP(s, size);
-#else
-    t = strndup(s, size);
-#endif
-    if (!t) throw std::bad_alloc();
+    auto t = allocString(size + 1);
+    memcpy(t, s, size);
+    t[size] = 0;
     return t;
 }
 

From 5b798f6caeab64854394e1250e30aab91241cf26 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 27 Nov 2022 12:57:18 +0100
Subject: [PATCH 099/112] Fix random client failures during GC server shutdown

We need to close the GC server socket before shutting down the active
GC client connections, otherwise a client may (re)connect and get
ECONNRESET. But also handle ECONNRESET for resilience.

Fixes random failures like

  GC socket disconnected
  connecting to '/tmp/nix-shell.y07M0H/nix-test/default/var/nix/gc-socket/socket'
  sending GC root '/tmp/nix-shell.y07M0H/nix-test/default/store/kb5yzija0f1x5xkqkgclrdzldxj6nnc6-non-blocking'
  reading GC root from client: error: unexpected EOF reading a line
  1 store paths deleted, 0.00 MiB freed
  error: reading from file: Connection reset by peer

in gc-non-blocking.sh.
---
 src/libstore/gc.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 9ef8972f3..5d91829f1 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -147,7 +147,7 @@ void LocalStore::addTempRoot(const StorePath & path)
         } catch (SysError & e) {
             /* The garbage collector may have exited, so we need to
                restart. */
-            if (e.errNo == EPIPE) {
+            if (e.errNo == EPIPE || e.errNo == ECONNRESET) {
                 debug("GC socket disconnected");
                 state->fdRootsSocket.close();
                 goto restart;
@@ -506,6 +506,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         Finally cleanup([&]() {
             debug("GC roots server shutting down");
+            fdServer.close();
             while (true) {
                 auto item = remove_begin(*connections.lock());
                 if (!item) break;

From f1b5c6876bc570ff9ac79410d8e47aadcb9aed52 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 27 Nov 2022 16:38:34 +0100
Subject: [PATCH 100/112] Add tests for auto-uid-allocation, uid-range and
 cgroups

---
 flake.nix                |  6 ++++
 tests/containers.nix     | 68 ++++++++++++++++++++++++++++++++++++
 tests/id-test.nix        |  8 +++++
 tests/systemd-nspawn.nix | 75 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 157 insertions(+)
 create mode 100644 tests/containers.nix
 create mode 100644 tests/id-test.nix
 create mode 100644 tests/systemd-nspawn.nix

diff --git a/flake.nix b/flake.nix
index cc2a48d9c..d9d01da10 100644
--- a/flake.nix
+++ b/flake.nix
@@ -506,6 +506,12 @@
           overlay = self.overlays.default;
         });
 
+        tests.containers = (import ./tests/containers.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          overlay = self.overlays.default;
+        });
+
         tests.setuid = nixpkgs.lib.genAttrs
           ["i686-linux" "x86_64-linux"]
           (system:
diff --git a/tests/containers.nix b/tests/containers.nix
new file mode 100644
index 000000000..d1e791b8c
--- /dev/null
+++ b/tests/containers.nix
@@ -0,0 +1,68 @@
+# Test whether we can run a NixOS container inside a Nix build using systemd-nspawn.
+{ nixpkgs, system, overlay }:
+
+with import (nixpkgs + "/nixos/lib/testing-python.nix") {
+  inherit system;
+  extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
+};
+
+makeTest ({
+  name = "containers";
+
+  nodes =
+    {
+      host =
+        { config, lib, pkgs, nodes, ... }:
+        { virtualisation.writableStore = true;
+          virtualisation.diskSize = 2048;
+          virtualisation.additionalPaths =
+            [ pkgs.stdenv
+              (import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel
+            ];
+          virtualisation.memorySize = 4096;
+          nix.binaryCaches = lib.mkForce [ ];
+          nix.extraOptions =
+            ''
+              extra-experimental-features = nix-command auto-allocate-uids
+              extra-system-features = uid-range
+            '';
+          nix.nixPath = [ "nixpkgs=${nixpkgs}" ];
+        };
+    };
+
+  testScript = { nodes }: ''
+    start_all()
+
+    host.succeed("nix --version >&2")
+
+    # Test that 'id' gives the expected result in various configurations.
+
+    # Existing UIDs, sandbox.
+    host.succeed("nix build --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1")
+    host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]")
+
+    # Existing UIDs, no sandbox.
+    host.succeed("nix build --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2")
+    host.succeed("[[ $(cat ./result) = 'uid=30001(nixbld1) gid=30000(nixbld) groups=30000(nixbld)' ]]")
+
+    # Auto-allocated UIDs, sandbox.
+    host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3")
+    host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]")
+
+    # Auto-allocated UIDs, no sandbox.
+    host.succeed("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4")
+    host.succeed("[[ $(cat ./result) = 'uid=872415232 gid=30000(nixbld) groups=30000(nixbld)' ]]")
+
+    # Auto-allocated UIDs, UID range, sandbox.
+    host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true")
+    host.succeed("[[ $(cat ./result) = 'uid=0(root) gid=0(root) groups=0(root)' ]]")
+
+    # Auto-allocated UIDs, UID range, no sandbox.
+    host.fail("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true")
+
+    # Run systemd-nspawn in a Nix build.
+    host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}")
+    host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]")
+  '';
+
+})
diff --git a/tests/id-test.nix b/tests/id-test.nix
new file mode 100644
index 000000000..8eb9d38f9
--- /dev/null
+++ b/tests/id-test.nix
@@ -0,0 +1,8 @@
+{ name, uidRange ? false }:
+
+with import <nixpkgs> {};
+
+runCommand name
+  { requiredSystemFeatures = if uidRange then ["uid-range"] else [];
+  }
+  "id; id > $out"
diff --git a/tests/systemd-nspawn.nix b/tests/systemd-nspawn.nix
new file mode 100644
index 000000000..49944eba3
--- /dev/null
+++ b/tests/systemd-nspawn.nix
@@ -0,0 +1,75 @@
+{ nixpkgs }:
+
+let
+
+  machine = { config, pkgs, ... }:
+    {
+      system.stateVersion = "22.05";
+      boot.isContainer = true;
+      systemd.services.console-getty.enable = false;
+      networking.dhcpcd.enable = false;
+
+      services.httpd = {
+        enable = true;
+        adminAddr = "nixos@example.org";
+      };
+
+      systemd.services.test = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "httpd.service" ];
+        script = ''
+          source /.env
+          echo "Hello World" > $out/msg
+          ls -lR /dev > $out/dev
+          ${pkgs.curl}/bin/curl -sS --fail http://localhost/ > $out/page.html
+        '';
+        unitConfig = {
+          FailureAction = "exit-force";
+          FailureActionExitStatus = 42;
+          SuccessAction = "exit-force";
+        };
+      };
+    };
+
+  config = (import (nixpkgs + "/nixos/lib/eval-config.nix") {
+    modules = [ machine ];
+  }).config;
+
+in
+
+with import nixpkgs {};
+
+runCommand "test"
+  { buildInputs = [ config.system.path ];
+    requiredSystemFeatures = [ "uid-range" ];
+    toplevel = config.system.build.toplevel;
+  }
+  ''
+    root=$(pwd)/root
+    mkdir -p $root $root/etc
+
+    export > $root/.env
+
+    # Make /run a tmpfs to shut up a systemd warning.
+    mkdir /run
+    mount -t tmpfs none /run
+    chmod 0700 /run
+
+    mount -t cgroup2 none /sys/fs/cgroup
+
+    mkdir -p $out
+
+    touch /etc/os-release
+    echo a5ea3f98dedc0278b6f3cc8c37eeaeac > /etc/machine-id
+
+    SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 \
+      ${config.systemd.package}/bin/systemd-nspawn \
+      --keep-unit \
+      -M ${config.networking.hostName} -D "$root" \
+      --register=no \
+      --resolv-conf=off \
+      --bind-ro=/nix/store \
+      --bind=$out \
+      --private-network \
+      $toplevel/init
+  ''

From fc1458561086a6cf2c1311294c9089785288aea3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 27 Nov 2022 18:58:21 +0100
Subject: [PATCH 101/112] Fix evaluation

---
 tests/systemd-nspawn.nix | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/systemd-nspawn.nix b/tests/systemd-nspawn.nix
index 49944eba3..424436b3f 100644
--- a/tests/systemd-nspawn.nix
+++ b/tests/systemd-nspawn.nix
@@ -31,13 +31,16 @@ let
       };
     };
 
-  config = (import (nixpkgs + "/nixos/lib/eval-config.nix") {
+  cfg = (import (nixpkgs + "/nixos/lib/eval-config.nix") {
     modules = [ machine ];
-  }).config;
+    system = "x86_64-linux";
+  });
+
+  config = cfg.config;
 
 in
 
-with import nixpkgs {};
+with cfg._module.args.pkgs;
 
 runCommand "test"
   { buildInputs = [ config.system.path ];

From 04ec1575177709dd3c1bd147cb8d0966237663d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Na=C3=AFm=20Favier?= <n@monade.li>
Date: Mon, 28 Nov 2022 10:38:23 +0100
Subject: [PATCH 102/112] repl: print a newline on ctrl-D

---
 src/libcmd/repl.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index bb254ff8d..ddf0d9a0a 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -270,6 +270,7 @@ void NixRepl::mainLoop()
             // ctrl-D should exit the debugger.
             state->debugStop = false;
             state->debugQuit = true;
+            std::cout << std::endl;
             break;
         }
         try {

From 9b35cc716b392dfb96a0e994349137f653b9c532 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Na=C3=AFm=20Favier?= <n@monade.li>
Date: Mon, 28 Nov 2022 14:59:06 +0100
Subject: [PATCH 103/112] use logger->cout

in order to avoid potential problems with the progress bar

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libcmd/repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index ddf0d9a0a..557952277 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -270,7 +270,7 @@ void NixRepl::mainLoop()
             // ctrl-D should exit the debugger.
             state->debugStop = false;
             state->debugQuit = true;
-            std::cout << std::endl;
+            logger->cout("");
             break;
         }
         try {

From ff12d1c1a1bb0dcea5a9ac6b8a5036d7e5dc11ca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Nov 2022 20:49:17 +0100
Subject: [PATCH 104/112] Check that auto-allocated UIDs don't clash with
 existing accounts

---
 src/libstore/lock.cc | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc
index 7459d837d..2858137d6 100644
--- a/src/libstore/lock.cc
+++ b/src/libstore/lock.cc
@@ -127,13 +127,10 @@ struct AutoUserLock : UserLock
     {
         settings.requireExperimentalFeature(Xp::AutoAllocateUids);
         assert(settings.startId > 0);
-        assert(settings.startId % maxIdsPerBuild == 0);
         assert(settings.uidCount % maxIdsPerBuild == 0);
         assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
         assert(nrIds <= maxIdsPerBuild);
 
-        // FIXME: check whether the id range overlaps any known users
-
         createDirs(settings.nixStateDir + "/userpool2");
 
         size_t nrSlots = settings.uidCount / maxIdsPerBuild;
@@ -150,11 +147,18 @@ struct AutoUserLock : UserLock
                 throw SysError("opening user lock '%s'", fnUserLock);
 
             if (lockFile(fd.get(), ltWrite, false)) {
+
+                auto firstUid = settings.startId + i * maxIdsPerBuild;
+
+                auto pw = getpwuid(firstUid);
+                if (pw)
+                    throw Error("auto-allocated UID %d clashes with existing user account '%s'", firstUid, pw->pw_name);
+
                 auto lock = std::make_unique<AutoUserLock>();
                 lock->fdUserLock = std::move(fd);
-                lock->firstUid = settings.startId + i * maxIdsPerBuild;
+                lock->firstUid = firstUid;
                 if (useChroot)
-                    lock->firstGid = lock->firstUid;
+                    lock->firstGid = firstUid;
                 else {
                     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
                     if (!gr)

From 67bcb99700a0da1395fa063d7c6586740b304598 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Nov 2022 21:54:02 +0100
Subject: [PATCH 105/112] Add a setting for enabling cgroups

---
 doc/manual/src/release-notes/rl-next.md     |  5 +++--
 src/libstore/build/local-derivation-goal.cc |  7 ++++++-
 src/libstore/globals.hh                     | 23 +++++++++++++++------
 tests/containers.nix                        |  2 +-
 4 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 4c91002fb..db2bd7419 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -43,10 +43,11 @@
 
   ```
   extra-experimental-features = cgroups
+  use-cgroups = true
   ```
 
-  to `nix.conf`. It is also automatically enabled for builds that
-  require the `uid-range` system feature.
+  to `nix.conf`. Cgroups are required for derivations that require the
+  `uid-range` system feature.
 
 * `nix build --json` now prints some statistics about top-level
   derivations, such as CPU statistics when cgroups are enabled.
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index d44694890..69a7df411 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -401,9 +401,14 @@ static void linkOrCopy(const Path & from, const Path & to)
 void LocalDerivationGoal::startBuilder()
 {
     if ((buildUser && buildUser->getUIDCount() != 1)
-        || settings.isExperimentalFeatureEnabled(Xp::Cgroups))
+        #if __linux__
+        || settings.useCgroups
+        #endif
+        )
     {
         #if __linux__
+        settings.requireExperimentalFeature(Xp::Cgroups);
+
         auto ourCgroups = getCgroups("/proc/self/cgroup");
         auto ourCgroup = ourCgroups[""];
         if (ourCgroup == "")
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 653d108aa..b40dcfa77 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -304,6 +304,17 @@ public:
         "id-count",
         "The number of UIDs/GIDs to use for dynamic ID allocation."};
 
+    #if __linux__
+    Setting<bool> useCgroups{
+        this, false, "use-cgroups",
+        R"(
+          Whether to execute builds inside cgroups. Cgroups are
+          enabled automatically for derivations that require the
+          `uid-range` system feature.
+        )"
+    };
+    #endif
+
     Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
         "Whether to impersonate a Linux 2.6 machine on newer kernels.",
         {"build-impersonate-linux-26"}};
@@ -592,10 +603,10 @@ public:
           cache) must have a signature by a trusted key. A trusted key is one
           listed in `trusted-public-keys`, or a public key counterpart to a
           private key stored in a file listed in `secret-key-files`.
-          
+
           Set to `false` to disable signature checking and trust all
           non-content-addressed paths unconditionally.
-          
+
           (Content-addressed paths are inherently trustworthy and thus
           unaffected by this configuration option.)
         )"};
@@ -681,7 +692,7 @@ public:
           is `root`.
 
           > **Warning**
-          > 
+          >
           > Adding a user to `trusted-users` is essentially equivalent to
           > giving that user root access to the system. For example, the user
           > can set `sandbox-paths` and thereby obtain read access to
@@ -771,13 +782,13 @@ public:
           The program executes with no arguments. The program's environment
           contains the following environment variables:
 
-            - `DRV_PATH`  
+            - `DRV_PATH`
               The derivation for the built paths.
 
               Example:
               `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
 
-            - `OUT_PATHS`  
+            - `OUT_PATHS`
               Output paths of the built derivation, separated by a space
               character.
 
@@ -815,7 +826,7 @@ public:
           documentation](https://ec.haxx.se/usingcurl-netrc.html).
 
           > **Note**
-          > 
+          >
           > This must be an absolute path, and `~` is not resolved. For
           > example, `~/.netrc` won't resolve to your home directory's
           > `.netrc`.
diff --git a/tests/containers.nix b/tests/containers.nix
index d1e791b8c..59e953c3b 100644
--- a/tests/containers.nix
+++ b/tests/containers.nix
@@ -23,7 +23,7 @@ makeTest ({
           nix.binaryCaches = lib.mkForce [ ];
           nix.extraOptions =
             ''
-              extra-experimental-features = nix-command auto-allocate-uids
+              extra-experimental-features = nix-command auto-allocate-uids cgroups
               extra-system-features = uid-range
             '';
           nix.nixPath = [ "nixpkgs=${nixpkgs}" ];

From 7dd3e1fec47b9dd6aa6a0b9a58962078a8499453 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Nov 2022 22:04:51 +0100
Subject: [PATCH 106/112] Add example

---
 doc/manual/src/release-notes/rl-next.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index db2bd7419..8b314b5f6 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -26,7 +26,8 @@
 * On Linux, Nix can now run builds in a user namespace where the build
   runs as root (UID 0) and has 65,536 UIDs available. This is
   primarily useful for running containers such as `systemd-nspawn`
-  inside a Nix build.
+  inside a Nix build. For an example, see
+  https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix.
 
   A build can enable this by requiring the `uid-range` system feature,
   i.e. by setting the derivation attribute

From dc61e1028d73db406d47c0fb979219f193b74777 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Nov 2022 22:00:44 +0000
Subject: [PATCH 107/112] Bump zeebe-io/backport-action from 0.0.8 to 0.0.9

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 0.0.8 to 0.0.9.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v0.0.8...v0.0.9)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 75be788ef..7568145b6 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v0.0.8
+        uses: zeebe-io/backport-action@v0.0.9
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From 46a6be28bef45640de5344a09d56add7068a9aa4 Mon Sep 17 00:00:00 2001
From: Rok Garbas <rok@garbas.si>
Date: Tue, 29 Nov 2022 10:01:46 +0000
Subject: [PATCH 108/112] Add nobody user/group to Nix docker image

---
 docker.nix | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docker.nix b/docker.nix
index bb2b4e7ff..203a06b53 100644
--- a/docker.nix
+++ b/docker.nix
@@ -36,6 +36,17 @@ let
       shell = "${pkgs.bashInteractive}/bin/bash";
       home = "/root";
       gid = 0;
+      groups = [ "root" ];
+      description = "System administrator";
+    };
+
+    nobody = {
+      uid = 65534;
+      shell = "${pkgs.shadow}/bin/nologin";
+      home = "/var/empty";
+      gid = 65534;
+      groups = [ "nobody" ];
+      description = "Unprivileged account (don't use!)";
     };
 
   } // lib.listToAttrs (
@@ -57,6 +68,7 @@ let
   groups = {
     root.gid = 0;
     nixbld.gid = 30000;
+    nobody.gid = 65534;
   };
 
   userToPasswd = (

From 4f762e2b023fd451fdbab0de8d6394dd7201640d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Nov 2022 13:10:53 +0100
Subject: [PATCH 109/112] Restore ownership of / for non-uid-range builds

---
 src/libstore/build/local-derivation-goal.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 69a7df411..359966288 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -646,8 +646,7 @@ void LocalDerivationGoal::startBuilder()
         if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
-        // FIXME: only make root writable for user namespace builds.
-        if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
+        if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1)
             throw SysError("cannot change ownership of '%1%'", chrootRootDir);
 
         /* Create a writable /tmp in the chroot.  Many builders need

From 0b092bd87f35e463fea66ddd40639e7b260680a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 Nov 2022 13:46:33 +0100
Subject: [PATCH 110/112] nix store make-content-addressed: Fix JSON
 construction

Fixes

  error: [json.exception.type_error.301] cannot create object from initializer list

in tests/fetchClosure.sh.
---
 src/nix/make-content-addressed.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc
index f2e4cefbe..f236bebd6 100644
--- a/src/nix/make-content-addressed.cc
+++ b/src/nix/make-content-addressed.cc
@@ -43,7 +43,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
                 assert(i != remappings.end());
                 jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second);
             }
-            std::cout << json::object({"rewrites", jsonRewrites}).dump();
+            std::cout << nlohmann::json{"rewrites", jsonRewrites}.dump();
         } else {
             for (auto & path : storePaths) {
                 auto i = remappings.find(path);

From 3f881e3378743039b32f3d3c650255b448732803 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 30 Nov 2022 23:40:53 +0100
Subject: [PATCH 111/112] add missing newline, for consistent formatting

---
 doc/manual/generate-manpage.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 057719e34..8c7c4d358 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -99,6 +99,7 @@ let
     in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
 
   parsedToplevel = builtins.fromJSON toplevel;
+  
   manpages = processCommand {
     command = "nix";
     details = parsedToplevel;

From f1e1ba9fe094a774f0fd05e537228e628d0bc8cb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 1 Dec 2022 16:29:09 +0100
Subject: [PATCH 112/112] Really fix 'nix store make-content-addressed --json'

https://hydra.nixos.org/log/mcgypcf9vj4n8vdmw7lj3l05c899v73w-nix-2.12.0pre20221201_16b03f0-x86_64-unknown-linux-musl.drv
---
 src/nix/make-content-addressed.cc |  6 ++++--
 tests/fetchClosure.sh             | 21 ++++++++++++---------
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc
index f236bebd6..d86b90fc7 100644
--- a/src/nix/make-content-addressed.cc
+++ b/src/nix/make-content-addressed.cc
@@ -37,13 +37,15 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
             StorePathSet(storePaths.begin(), storePaths.end()));
 
         if (json) {
-            nlohmann::json jsonRewrites = json::object();
+            auto jsonRewrites = json::object();
             for (auto & path : storePaths) {
                 auto i = remappings.find(path);
                 assert(i != remappings.end());
                 jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second);
             }
-            std::cout << nlohmann::json{"rewrites", jsonRewrites}.dump();
+            auto json = json::object();
+            json["rewrites"] = jsonRewrites;
+            std::cout << json.dump();
         } else {
             for (auto & path : storePaths) {
                 auto i = remappings.find(path);
diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh
index 44050c878..d88c55c3c 100644
--- a/tests/fetchClosure.sh
+++ b/tests/fetchClosure.sh
@@ -1,7 +1,6 @@
 source common.sh
 
 enableFeatures "fetch-closure"
-needLocalStore "'--no-require-sigs' can’t be used with the daemon"
 
 clearStore
 clearCacheCache
@@ -28,15 +27,19 @@ clearStore
 [ ! -e $nonCaPath ]
 [ -e $caPath ]
 
-# In impure mode, we can use non-CA paths.
-[[ $(nix eval --raw --no-require-sigs --impure --expr "
-  builtins.fetchClosure {
-    fromStore = \"file://$cacheDir\";
-    fromPath = $nonCaPath;
-  }
-") = $nonCaPath ]]
+if [[ "$NIX_REMOTE" != "daemon" ]]; then
 
-[ -e $nonCaPath ]
+    # In impure mode, we can use non-CA paths.
+    [[ $(nix eval --raw --no-require-sigs --impure --expr "
+      builtins.fetchClosure {
+        fromStore = \"file://$cacheDir\";
+        fromPath = $nonCaPath;
+      }
+    ") = $nonCaPath ]]
+
+    [ -e $nonCaPath ]
+
+fi
 
 # 'toPath' set to empty string should fail but print the expected path.
 nix eval -v --json --expr "