Merge remote-tracking branch 'nixos/master'

.gitignore

@@ -90,6 +90,7 @@ perl/Makefile.config
 
 /misc/systemd/nix-daemon.service
 /misc/systemd/nix-daemon.socket
+/misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 
 /src/resolve-system-dependencies/resolve-system-dependencies

doc/manual/src/installation/installing-binary.md

@@ -84,7 +84,9 @@ The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
 The installer will first back up these files with a `.backup-before-nix`
 extension. The installer will also create `/etc/profile.d/nix.sh`.
 
-You can uninstall Nix with the following commands:
+## Uninstalling
+
+### Linux
 
 ```console
 sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
@@ -95,15 +97,95 @@ sudo systemctl stop nix-daemon.service
 sudo systemctl disable nix-daemon.socket
 sudo systemctl disable nix-daemon.service
 sudo systemctl daemon-reload
-
-# If you are on macOS, you will need to run:
-sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 ```
 
 There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
 and `/etc/zshrc` which you may remove.
 
+### macOS
+
+1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing
+   `nix-daemon.sh`, which should look like this:
+
+   ```bash
+   # Nix
+   if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then
+     . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh'
+   fi
+   # End Nix
+   ```
+
+   If these files haven't been altered since installing Nix you can simply put
+   the backups back in place:
+
+   ```console
+   sudo mv /etc/zshrc.backup-before-nix /etc/zshrc
+   sudo mv /etc/bashrc.backup-before-nix /etc/bashrc
+   ```
+
+   This will stop shells from sourcing the file and bringing everything you
+   installed using Nix in scope.
+
+2. Stop and remove the Nix daemon services:
+
+   ```console
+   sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+   sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+   sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist
+   sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist
+   ```
+
+   This stops the Nix daemon and prevents it from being started next time you
+   boot the system.
+
+3. Remove the `nixbld` group and the `_nixbuildN` users:
+
+   ```console
+   sudo dscl . -delete /Groups/nixbld
+   for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done
+   ```
+
+   This will remove all the build users that no longer serve a purpose.
+
+4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store
+   volume on `/nix`, which looks like this,
+   `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic
+   mounting of the Nix Store volume.
+
+5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only
+   line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`.
+   This will prevent the creation of the empty `/nix` directory to provide a
+   mountpoint for the Nix Store volume.
+
+6. Remove the files Nix added to your system:
+
+   ```console
+   sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+   ```
+
+   This gets rid of any data Nix may have created except for the store which is
+   removed next.
+
+7. Remove the Nix Store volume:
+   
+   ```console
+   sudo diskutil apfs deleteVolume /nix
+   ```
+
+   This will remove the Nix Store volume and everything that was added to the
+   store.
+
+> **Note**
+> 
+> After you complete the steps here, you will still have an empty `/nix`
+> directory. This is an expected sign of a successful uninstall. The empty
+> `/nix` directory will disappear the next time you reboot.
+>
+> You do not have to reboot to finish uninstalling Nix. The uninstall is
+> complete. macOS (Catalina+) directly controls root directories and its
+> read-only root will prevent you from manually deleting the empty `/nix`
+> mountpoint.
+
 # macOS Installation <a name="sect-macos-installation-change-store-prefix"></a><a name="sect-macos-installation-encrypted-volume"></a><a name="sect-macos-installation-symlink"></a><a name="sect-macos-installation-recommended-notes"></a>
 <!-- Note: anchors above to catch permalinks to old explanations -->
 

doc/manual/src/release-notes/rl-next.md

@@ -1,3 +1,16 @@
 # Release X.Y (202?-??-??)
 
 * Various nix commands can now read expressions from stdin with `--file -`.
+
+* `nix store make-content-addressable` has been renamed to `nix store
+  make-content-addressed`.
+
+* New experimental builtin function `builtins.fetchClosure` that
+  copies a closure from a binary cache at evaluation time and rewrites
+  it to content-addressed form (if it isn't already). Like
+  `builtins.storePath`, this allows importing pre-built store paths;
+  the difference is that it doesn't require the user to configure
+  binary caches and trusted public keys.
+
+  This function is only available if you enable the experimental
+  feature `fetch-closure`.

misc/systemd/local.mk

@@ -1,7 +1,8 @@
 ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
+  $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644)))
 
-  clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service
+  clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf
 
 endif

misc/systemd/nix-daemon.conf.in

@@ -0,0 +1 @@
+d @localstatedir@/nix/daemon-socket 0755 root root - -

misc/systemd/nix-daemon.service.in

@@ -3,6 +3,7 @@ Description=Nix Daemon
 Documentation=man:nix-daemon https://nixos.org/manual
 RequiresMountsFor=@storedir@
 RequiresMountsFor=@localstatedir@
+RequiresMountsFor=@localstatedir@/nix/db
 ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 
 [Service]

scripts/install-multi-user.sh

@@ -739,7 +739,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              cp -RLp ./store/* "$NIX_ROOT/store/"
+              cp -RPp ./store/* "$NIX_ROOT/store/"
 
         _sudo "to make the new store non-writable at $NIX_ROOT/store" \
               chmod -R ugo-w "$NIX_ROOT/store/"

scripts/install-systemd-multi-user.sh

@@ -9,6 +9,8 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
 readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
 
+readonly TMPFILES_SRC=/lib/tmpfiles.d/nix-daemon.conf
+readonly TMPFILES_DEST=/etc/tmpfiles.d/nix-daemon.conf
 
 # Path for the systemd override unit file to contain the proxy settings
 readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
@@ -83,6 +85,13 @@ EOF
 poly_configure_nix_daemon_service() {
     if [ -e /run/systemd/system ]; then
         task "Setting up the nix-daemon systemd service"
+
+        _sudo "to create the nix-daemon tmpfiles config" \
+              ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST
+
+        _sudo "to run systemd-tmpfiles once to pick that path up" \
+             systemd-tmpfiles create --prefix=/nix/var/nix
+
         _sudo "to set up the nix-daemon service" \
               systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
 

src/libcmd/command.cc

@@ -204,7 +204,8 @@ Strings editorFor(const Pos & pos)
     if (pos.line > 0 && (
         editor.find("emacs") != std::string::npos ||
         editor.find("nano") != std::string::npos ||
-        editor.find("vim") != std::string::npos))
+        editor.find("vim") != std::string::npos ||
+        editor.find("kak") != std::string::npos))
         args.push_back(fmt("+%d", pos.line));
     args.push_back(pos.file);
     return args;

src/libcmd/installables.cc

@@ -1002,55 +1002,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
+BuiltPaths Installable::build(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode)
 {
     BuiltPaths res;
-    for (const auto & b : hopefullyBuiltPaths)
+    for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
-        std::visit(
+        res.push_back(builtPath);
-            overloaded{
-                [&](const DerivedPath::Opaque & bo) {
-                    res.push_back(BuiltPath::Opaque{bo.path});
-                },
-                [&](const DerivedPath::Built & bfd) {
-                    OutputPathMap outputs;
-                    auto drv = evalStore->readDerivation(bfd.drvPath);
-                    auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
-                    auto drvOutputs = drv.outputsAndOptPaths(*store);
-                    for (auto & output : bfd.outputs) {
-                        if (!outputHashes.count(output))
-                            throw Error(
-                                "the derivation '%s' doesn't have an output named '%s'",
-                                store->printStorePath(bfd.drvPath), output);
-                        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
-                            auto outputId =
-                                DrvOutput{outputHashes.at(output), output};
-                            auto realisation =
-                                store->queryRealisation(outputId);
-                            if (!realisation)
-                                throw Error(
-                                    "cannot operate on an output of unbuilt "
-                                    "content-addressed derivation '%s'",
-                                    outputId.to_string());
-                            outputs.insert_or_assign(
-                                output, realisation->outPath);
-                        } else {
-                            // If ca-derivations isn't enabled, assume that
-                            // the output path is statically known.
-                            assert(drvOutputs.count(output));
-                            assert(drvOutputs.at(output).second);
-                            outputs.insert_or_assign(
-                                output, *drvOutputs.at(output).second);
-                        }
-                    }
-                    res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
-                },
-            },
-            b.raw());
-
     return res;
 }
 
-BuiltPaths Installable::build(
+std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -1061,39 +1026,93 @@ BuiltPaths Installable::build(
         settings.readOnlyMode = true;
 
     std::vector<DerivedPath> pathsToBuild;
+    std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap;
 
     for (auto & i : installables) {
-        auto b = i->toDerivedPaths();
+        for (auto b : i->toDerivedPaths()) {
-        pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
+            pathsToBuild.push_back(b);
+            backmap[b].push_back(i);
+        }
     }
 
+    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
+
     switch (mode) {
+
     case Realise::Nothing:
     case Realise::Derivation:
         printMissing(store, pathsToBuild, lvlError);
-        return getBuiltPaths(evalStore, store, pathsToBuild);
+
+        for (auto & path : pathsToBuild) {
+            for (auto & installable : backmap[path]) {
+                std::visit(overloaded {
+                    [&](const DerivedPath::Built & bfd) {
+                        OutputPathMap outputs;
+                        auto drv = evalStore->readDerivation(bfd.drvPath);
+                        auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
+                        auto drvOutputs = drv.outputsAndOptPaths(*store);
+                        for (auto & output : bfd.outputs) {
+                            if (!outputHashes.count(output))
+                                throw Error(
+                                    "the derivation '%s' doesn't have an output named '%s'",
+                                    store->printStorePath(bfd.drvPath), output);
+                            if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+                                DrvOutput outputId { outputHashes.at(output), output };
+                                auto realisation = store->queryRealisation(outputId);
+                                if (!realisation)
+                                    throw Error(
+                                        "cannot operate on an output of unbuilt "
+                                        "content-addressed derivation '%s'",
+                                        outputId.to_string());
+                                outputs.insert_or_assign(output, realisation->outPath);
+                            } else {
+                                // If ca-derivations isn't enabled, assume that
+                                // the output path is statically known.
+                                assert(drvOutputs.count(output));
+                                assert(drvOutputs.at(output).second);
+                                outputs.insert_or_assign(
+                                    output, *drvOutputs.at(output).second);
+                            }
+                        }
+                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                    },
+                    [&](const DerivedPath::Opaque & bo) {
+                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                    },
+                }, path.raw());
+            }
+        }
+
+        break;
+
     case Realise::Outputs: {
-        BuiltPaths res;
         for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
             if (!buildResult.success())
                 buildResult.rethrow();
-            std::visit(overloaded {
+
-                [&](const DerivedPath::Built & bfd) {
+            for (auto & installable : backmap[buildResult.path]) {
-                    std::map<std::string, StorePath> outputs;
+                std::visit(overloaded {
-                    for (auto & path : buildResult.builtOutputs)
+                    [&](const DerivedPath::Built & bfd) {
-                        outputs.emplace(path.first.outputName, path.second.outPath);
+                        std::map<std::string, StorePath> outputs;
-                    res.push_back(BuiltPath::Built { bfd.drvPath, outputs });
+                        for (auto & path : buildResult.builtOutputs)
-                },
+                            outputs.emplace(path.first.outputName, path.second.outPath);
-                [&](const DerivedPath::Opaque & bo) {
+                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
-                    res.push_back(BuiltPath::Opaque { bo.path });
+                    },
-                },
+                    [&](const DerivedPath::Opaque & bo) {
-            }, buildResult.path.raw());
+                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                    },
+                }, buildResult.path.raw());
+            }
         }
-        return res;
+
+        break;
     }
+
     default:
         assert(false);
     }
+
+    return res;
 }
 
 BuiltPaths Installable::toBuiltPaths(

src/libcmd/installables.hh

@@ -98,6 +98,13 @@ struct Installable
         const std::vector<std::shared_ptr<Installable>> & installables,
         BuildMode bMode = bmNormal);
 
+    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        const std::vector<std::shared_ptr<Installable>> & installables,
+        BuildMode bMode = bmNormal);
+
     static std::set<StorePath> toStorePaths(
         ref<Store> evalStore,
         ref<Store> store,
@@ -185,9 +192,4 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake);
 
-BuiltPaths getBuiltPaths(
-    ref<Store> evalStore,
-    ref<Store> store,
-    const DerivedPaths & hopefullyBuiltPaths);
-
 }

src/libexpr/eval-cache.cc

@@ -21,6 +21,8 @@ struct AttrDb
 {
     std::atomic_bool failed{false};
 
+    const Store & cfg;
+
     struct State
     {
         SQLite db;
@@ -33,8 +35,9 @@ struct AttrDb
 
     std::unique_ptr<Sync<State>> _state;
 
-    AttrDb(const Hash & fingerprint)
+    AttrDb(const Store & cfg, const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
+        : cfg(cfg)
+        , _state(std::make_unique<Sync<State>>())
     {
         auto state(_state->lock());
 
@@ -254,10 +257,10 @@ struct AttrDb
                 return {{rowId, attrs}};
             }
             case AttrType::String: {
-                std::vector<std::pair<Path, std::string>> context;
+                NixStringContext context;
                 if (!queryAttribute.isNull(3))
                     for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(s));
+                        context.push_back(decodeContext(cfg, s));
                 return {{rowId, string_t{queryAttribute.getStr(2), context}}};
             }
             case AttrType::Bool:
@@ -274,10 +277,10 @@ struct AttrDb
     }
 };
 
-static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+static std::shared_ptr<AttrDb> makeAttrDb(const Store & cfg, const Hash & fingerprint)
 {
     try {
-        return std::make_shared<AttrDb>(fingerprint);
+        return std::make_shared<AttrDb>(cfg, fingerprint);
     } catch (SQLiteError &) {
         ignoreException();
         return nullptr;
@@ -288,7 +291,7 @@ EvalCache::EvalCache(
     std::optional<std::reference_wrapper<const Hash>> useCache,
     EvalState & state,
     RootLoader rootLoader)
-    : db(useCache ? makeAttrDb(*useCache) : nullptr)
+    : db(useCache ? makeAttrDb(*state.store, *useCache) : nullptr)
     , state(state)
     , rootLoader(rootLoader)
 {
@@ -546,7 +549,7 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+                    if (!root->state.store->isValidPath(c.first)) {
                         valid = false;
                         break;
                     }
@@ -563,7 +566,7 @@ string_t AttrCursor::getStringWithContext()
     auto & v = forceValue();
 
     if (v.type() == nString)
-        return {v.string.s, v.getContext()};
+        return {v.string.s, v.getContext(*root->state.store)};
     else if (v.type() == nPath)
         return {v.path, {}};
     else

src/libexpr/eval-cache.hh

@@ -52,7 +52,7 @@ struct misc_t {};
 struct failed_t {};
 typedef uint64_t AttrId;
 typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+typedef std::pair<std::string, NixStringContext> string_t;
 
 typedef std::variant<
     std::vector<Symbol>,

src/libexpr/eval.cc

@@ -96,20 +96,20 @@ RootValue allocRootValue(Value * v)
 }
 
 
-void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v)
+void Value::print(std::ostream & str, std::set<const void *> * seen) const
 {
     checkInterrupt();
 
-    switch (v.internalType) {
+    switch (internalType) {
     case tInt:
-        str << v.integer;
+        str << integer;
         break;
     case tBool:
-        str << (v.boolean ? "true" : "false");
+        str << (boolean ? "true" : "false");
         break;
     case tString:
         str << "\"";
-        for (const char * i = v.string.s; *i; i++)
+        for (const char * i = string.s; *i; i++)
             if (*i == '\"' || *i == '\\') str << "\\" << *i;
             else if (*i == '\n') str << "\\n";
             else if (*i == '\r') str << "\\r";
@@ -119,19 +119,19 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
         str << "\"";
         break;
     case tPath:
-        str << v.path; // !!! escaping?
+        str << path; // !!! escaping?
         break;
     case tNull:
         str << "null";
         break;
     case tAttrs: {
-        if (!v.attrs->empty() && !seen.insert(v.attrs).second)
+        if (seen && !attrs->empty() && !seen->insert(attrs).second)
-            str << "<REPEAT>";
+            str << "«repeated»";
         else {
             str << "{ ";
-            for (auto & i : v.attrs->lexicographicOrder()) {
+            for (auto & i : attrs->lexicographicOrder()) {
                 str << i->name << " = ";
-                printValue(str, seen, *i->value);
+                i->value->print(str, seen);
                 str << "; ";
             }
             str << "}";
@@ -141,12 +141,12 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
     case tList1:
     case tList2:
     case tListN:
-        if (v.listSize() && !seen.insert(v.listElems()).second)
+        if (seen && listSize() && !seen->insert(listElems()).second)
-            str << "<REPEAT>";
+            str << "«repeated»";
         else {
             str << "[ ";
-            for (auto v2 : v.listItems()) {
+            for (auto v2 : listItems()) {
-                printValue(str, seen, *v2);
+                v2->print(str, seen);
                 str << " ";
             }
             str << "]";
@@ -166,10 +166,10 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
         str << "<PRIMOP-APP>";
         break;
     case tExternal:
-        str << *v.external;
+        str << *external;
         break;
     case tFloat:
-        str << v.fpoint;
+        str << fpoint;
         break;
     default:
         abort();
@@ -177,10 +177,16 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
 }
 
 
-std::ostream & operator << (std::ostream & str, const Value & v)
+void Value::print(std::ostream & str, bool showRepeated) const
 {
     std::set<const void *> seen;
-    printValue(str, seen, v);
+    print(str, showRepeated ? nullptr : &seen);
+}
+
+
+std::ostream & operator << (std::ostream & str, const Value & v)
+{
+    v.print(str, false);
     return str;
 }
 
@@ -501,23 +507,6 @@ EvalState::~EvalState()
 }
 
 
-void EvalState::requireExperimentalFeatureOnEvaluation(
-    const ExperimentalFeature & feature,
-    const std::string_view fName,
-    const Pos & pos)
-{
-    if (!settings.isExperimentalFeatureEnabled(feature)) {
-        throw EvalError({
-            .msg = hintfmt(
-                "Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.",
-                feature,
-                fName
-            ),
-            .errPos = pos
-        });
-    }
-}
-
 void EvalState::allowPath(const Path & path)
 {
     if (allowedPaths)
@@ -1903,13 +1892,22 @@ std::string_view EvalState::forceString(Value & v, const Pos & pos)
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
    name>. */
-std::pair<std::string, std::string> decodeContext(std::string_view s)
+NixStringContextElem decodeContext(const Store & store, std::string_view s)
 {
     if (s.at(0) == '!') {
         size_t index = s.find("!", 1);
-        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+        return {
+            store.parseStorePath(s.substr(index + 1)),
+            std::string(s.substr(1, index - 1)),
+        };
     } else
-        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+        return {
+            store.parseStorePath(
+                s.at(0) == '/'
+                ? s
+                : s.substr(1)),
+            "",
+        };
 }
 
 
@@ -1921,13 +1919,13 @@ void copyContext(const Value & v, PathSet & context)
 }
 
 
-std::vector<std::pair<Path, std::string>> Value::getContext()
+NixStringContext Value::getContext(const Store & store)
 {
-    std::vector<std::pair<Path, std::string>> res;
+    NixStringContext res;
     assert(internalType == tString);
     if (string.context)
         for (const char * * p = string.context; *p; ++p)
-            res.push_back(decodeContext(*p));
+            res.push_back(decodeContext(store, *p));
     return res;
 }
 

src/libexpr/eval.hh

@@ -149,12 +149,6 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    void requireExperimentalFeatureOnEvaluation(
-        const ExperimentalFeature &,
-        const std::string_view fName,
-        const Pos & pos
-    );
-
     void addToSearchPath(const std::string & s);
 
     SearchPath getSearchPath() { return searchPath; }
@@ -430,7 +424,7 @@ std::string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
    name>. */
-std::pair<std::string, std::string> decodeContext(std::string_view s);
+NixStringContextElem decodeContext(const Store & store, std::string_view s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);

src/libexpr/flake/flake.cc

@@ -706,8 +706,6 @@ void callFlake(EvalState & state,
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
-
     std::string flakeRefS(state.forceStringNoCtx(*args[0], pos));
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
     if (evalSettings.pureEval && !flakeRef.input.isLocked())
@@ -723,7 +721,30 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
         v);
 }
 
-static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
+static RegisterPrimOp r2({
+    .name =  "__getFlake",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
+
+      ```nix
+      (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
+      ```
+
+      Unless impure evaluation is allowed (`--impure`), the flake reference
+      must be "locked", e.g. contain a Git revision or content hash. An
+      example of an unlocked usage is:
+
+      ```nix
+      (builtins.getFlake "github:edolstra/dwarffs").rev
+      ```
+
+      This function is only available if you enable the experimental feature
+      `flakes`.
+    )",
+    .fun = prim_getFlake,
+    .experimentalFeature = Xp::Flakes,
+});
 
 }
 

src/libexpr/lexer.l

@@ -28,6 +28,13 @@ using namespace nix;
 
 namespace nix {
 
+static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
+{
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
+}
+
+#define CUR_POS makeCurPos(*yylloc, data)
+
 // backup to recover from yyless(0)
 YYLTYPE prev_yylloc;
 
@@ -37,7 +44,6 @@ static void initLoc(YYLTYPE * loc)
     loc->first_column = loc->last_column = 1;
 }
 
-
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
     prev_yylloc = *loc;
@@ -147,14 +153,20 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError("invalid integer '%1%'", yytext);
+                  throw ParseError({
+                      .msg = hintfmt("invalid integer '%1%'", yytext),
+                      .errPos = CUR_POS,
+                  });
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError("invalid float '%1%'", yytext);
+                  throw ParseError({
+                      .msg = hintfmt("invalid float '%1%'", yytext),
+                      .errPos = CUR_POS,
+                  });
               return FLOAT;
             }
 
@@ -280,7 +292,10 @@ or          { return OR_KW; }
 
 <INPATH_SLASH>{ANY} |
 <INPATH_SLASH><<EOF>> {
-  throw ParseError("path has a trailing slash");
+  throw ParseError({
+      .msg = hintfmt("path has a trailing slash"),
+      .errPos = CUR_POS,
+  });
 }
 
 {SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }

src/libexpr/primops.cc

@@ -43,8 +43,8 @@ StringMap EvalState::realiseContext(const PathSet & context)
     StringMap res;
 
     for (auto & i : context) {
-        auto [ctxS, outputName] = decodeContext(i);
+        auto [ctx, outputName] = decodeContext(*store, i);
-        auto ctx = store->parseStorePath(ctxS);
+        auto ctxS = store->printStorePath(ctx);
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
         if (!outputName.empty() && ctx.isDerivation()) {
@@ -694,7 +694,32 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
 
 static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
     .name = "__genericClosure",
+    .args = {"attrset"},
     .arity = 1,
+    .doc = R"(
+      Take an *attrset* with values named `startSet` and `operator` in order to
+      return a *list of attrsets* by starting with the `startSet`, recursively
+      applying the `operator` function to each element. The *attrsets* in the
+      `startSet` and produced by the `operator` must each contain value named
+      `key` which are comparable to each other. The result is produced by
+      repeatedly calling the operator for each element encountered with a
+      unique key, terminating when no new elements are produced. For example,
+
+      ```
+      builtins.genericClosure {
+        startSet = [ {key = 5;} ];
+        operator = item: [{
+          key = if (item.key / 2 ) * 2 == item.key
+               then item.key / 2
+               else 3 * item.key + 1;
+        }];
+      }
+      ```
+      evaluates to
+      ```
+      [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
+      ```
+      )",
     .fun = prim_genericClosure,
 });
 
@@ -1118,8 +1143,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
-            auto ctx = decodeContext(path);
+            auto ctx = decodeContext(*state.store, path);
-            drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second);
+            drv.inputDrvs[ctx.first].insert(ctx.second);
         }
 
         /* Otherwise it's a source file. */
@@ -1197,34 +1222,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 DerivationOutput::Deferred { });
         }
 
-        // Regular, non-CA derivation should always return a single hash and not
+        auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
-        // hash per output.
+        switch (hashModulo.kind) {
-        auto hashModulo = hashDerivationModulo(*state.store, drv, true);
+        case DrvHash::Kind::Regular:
-        std::visit(overloaded {
+            for (auto & i : outputs) {
-            [&](const DrvHash & drvHash) {
+                auto h = hashModulo.hashes.at(i);
-                auto & h = drvHash.hash;
+                auto outPath = state.store->makeOutputPath(i, h, drvName);
-                switch (drvHash.kind) {
+                drv.env[i] = state.store->printStorePath(outPath);
-                case DrvHash::Kind::Deferred:
+                drv.outputs.insert_or_assign(
-                    /* Outputs already deferred, nothing to do */
+                    i,
-                    break;
+                    DerivationOutputInputAddressed {
-                case DrvHash::Kind::Regular:
+                        .path = std::move(outPath),
-                    for (auto & [outputName, output] : drv.outputs) {
+                    });
-                        auto outPath = state.store->makeOutputPath(outputName, h, drvName);
+            }
-                        drv.env[outputName] = state.store->printStorePath(outPath);
+            break;
-                        output = DerivationOutput::InputAddressed {
+            ;
-                            .path = std::move(outPath),
+        case DrvHash::Kind::Deferred:
-                        };
+            for (auto & i : outputs) {
-                    }
+                drv.outputs.insert_or_assign(i, DerivationOutputDeferred {});
-                    break;
+            }
-                }
+        }
-            },
-            [&](const CaOutputHashes &) {
-                // Shouldn't happen as the toplevel derivation is not CA.
-                assert(false);
-            },
-        },
-        hashModulo.raw());
-
     }
 
     /* Write the resulting term into the Nix store directory. */
@@ -3789,7 +3806,7 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
         .name = name,
         .args = {},
         .arity = arity,
-        .fun = fun
+        .fun = fun,
     });
 }
 
@@ -3864,13 +3881,17 @@ void EvalState::createBaseEnv()
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
-            addPrimOp({
+            if (!primOp.experimentalFeature
-                .fun = primOp.fun,
+                || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature))
-                .arity = std::max(primOp.args.size(), primOp.arity),
+            {
-                .name = symbols.create(primOp.name),
+                addPrimOp({
-                .args = primOp.args,
+                    .fun = primOp.fun,
-                .doc = primOp.doc,
+                    .arity = std::max(primOp.args.size(), primOp.arity),
-            });
+                    .name = symbols.create(primOp.name),
+                    .args = primOp.args,
+                    .doc = primOp.doc,
+                });
+            }
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */

src/libexpr/primops.hh

@@ -16,6 +16,7 @@ struct RegisterPrimOp
         size_t arity = 0;
         const char * doc;
         PrimOpFun fun;
+        std::optional<ExperimentalFeature> experimentalFeature;
     };
 
     typedef std::vector<Info> PrimOps;
@@ -35,6 +36,7 @@ struct RegisterPrimOp
 /* These primops are disabled without enableNativeCode, but plugins
    may wish to use them in limited contexts without globally enabling
    them. */
+
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
 

src/libexpr/primops/context.cc

@@ -83,8 +83,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
             drv = std::string(p, 1);
             path = &drv;
         } else if (p.at(0) == '!') {
-            std::pair<std::string, std::string> ctx = decodeContext(p);
+            NixStringContextElem ctx = decodeContext(*state.store, p);
-            drv = ctx.first;
+            drv = state.store->printStorePath(ctx.first);
             output = ctx.second;
             path = &drv;
         }

src/libexpr/primops/fetchClosure.cc

@@ -0,0 +1,154 @@
+#include "primops.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "url.hh"
+
+namespace nix {
+
+static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[0], pos);
+
+    std::optional<std::string> fromStoreUrl;
+    std::optional<StorePath> fromPath;
+    bool toCA = false;
+    std::optional<StorePath> toPath;
+
+    for (auto & attr : *args[0]->attrs) {
+        if (attr.name == "fromPath") {
+            PathSet context;
+            fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+        }
+
+        else if (attr.name == "toPath") {
+            state.forceValue(*attr.value, *attr.pos);
+            toCA = true;
+            if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+                PathSet context;
+                toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+            }
+        }
+
+        else if (attr.name == "fromStore")
+            fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
+
+        else
+            throw Error({
+                .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name),
+                .errPos = pos
+            });
+    }
+
+    if (!fromPath)
+        throw Error({
+            .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
+            .errPos = pos
+        });
+
+    if (!fromStoreUrl)
+        throw Error({
+            .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
+            .errPos = pos
+        });
+
+    auto parsedURL = parseURL(*fromStoreUrl);
+
+    if (parsedURL.scheme != "http" &&
+        parsedURL.scheme != "https" &&
+        !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
+        throw Error({
+            .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
+            .errPos = pos
+        });
+
+    auto fromStore = openStore(parsedURL.to_string());
+
+    if (toCA) {
+        if (!toPath || !state.store->isValidPath(*toPath)) {
+            auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
+            auto i = remappings.find(*fromPath);
+            assert(i != remappings.end());
+            if (toPath && *toPath != i->second)
+                throw Error({
+                    .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+                        state.store->printStorePath(*fromPath),
+                        state.store->printStorePath(i->second),
+                        state.store->printStorePath(*toPath)),
+                    .errPos = pos
+                });
+            if (!toPath)
+                throw Error({
+                    .msg = hintfmt(
+                        "rewriting '%s' to content-addressed form yielded '%s'; "
+                        "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+                        state.store->printStorePath(*fromPath),
+                        state.store->printStorePath(i->second)),
+                    .errPos = pos
+                });
+        }
+    } else {
+        copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+        toPath = fromPath;
+    }
+
+    /* In pure mode, require a CA path. */
+    if (evalSettings.pureEval) {
+        auto info = state.store->queryPathInfo(*toPath);
+        if (!info->isContentAddressed(*state.store))
+            throw Error({
+                .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
+                    state.store->printStorePath(*toPath)),
+                .errPos = pos
+            });
+    }
+
+    auto toPathS = state.store->printStorePath(*toPath);
+    v.mkString(toPathS, {toPathS});
+}
+
+static RegisterPrimOp primop_fetchClosure({
+    .name = "__fetchClosure",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a Nix store closure from a binary cache, rewriting it into
+      content-addressed form. For example,
+
+      ```nix
+      builtins.fetchClosure {
+        fromStore = "https://cache.nixos.org";
+        fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+        toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+      }
+      ```
+
+      fetches `/nix/store/r2jd...` from the specified binary cache,
+      and rewrites it into the content-addressed store path
+      `/nix/store/ldbh...`.
+
+      If `fromPath` is already content-addressed, or if you are
+      allowing impure evaluation (`--impure`), then `toPath` may be
+      omitted.
+
+      To find out the correct value for `toPath` given a `fromPath`,
+      you can use `nix store make-content-addressed`:
+
+      ```console
+      # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
+      rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
+      ```
+
+      This function is similar to `builtins.storePath` in that it
+      allows you to use a previously built store path in a Nix
+      expression. However, it is more reproducible because it requires
+      specifying a binary cache from which the path can be fetched.
+      Also, requiring a content-addressed final store path avoids the
+      need for users to configure binary cache public keys.
+
+      This function is only available if you enable the experimental
+      feature `fetch-closure`.
+    )",
+    .fun = prim_fetchClosure,
+    .experimentalFeature = Xp::FetchClosure,
+});
+
+}

src/libexpr/primops/fetchTree.cc

@@ -145,7 +145,7 @@ static void fetchTree(
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 throw Error({
-                    .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+                    .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"),
                     .errPos = pos
                 });
 
@@ -329,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({
     .fun = prim_fetchTarball,
 });
 
-static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
 }

src/libexpr/value.hh

@@ -57,6 +57,8 @@ struct ExprLambda;
 struct PrimOp;
 class Symbol;
 struct Pos;
+class StorePath;
+class Store;
 class EvalState;
 class XMLWriter;
 class JSONPlaceholder;
@@ -64,6 +66,8 @@ class JSONPlaceholder;
 
 typedef int64_t NixInt;
 typedef double NixFloat;
+typedef std::pair<StorePath, std::string> NixStringContextElem;
+typedef std::vector<NixStringContextElem> NixStringContext;
 
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
@@ -115,10 +119,13 @@ private:
     InternalType internalType;
 
     friend std::string showType(const Value & v);
-    friend void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v);
+
+    void print(std::ostream & str, std::set<const void *> * seen) const;
 
 public:
 
+    void print(std::ostream & str, bool showRepeated = false) const;
+
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
     // needed by callers into methods of this type
@@ -368,7 +375,7 @@ public:
        non-trivial. */
     bool isTrivial() const;
 
-    std::vector<std::pair<Path, std::string>> getContext();
+    NixStringContext getContext(const Store &);
 
     auto listItems()
     {

src/libfetchers/github.cc

@@ -390,7 +390,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
 
             ref_uri = line.substr(ref_index+5, line.length()-1);
         } else
-            ref_uri = fmt("refs/heads/%s", ref);
+            ref_uri = fmt("refs/(heads|tags)/%s", ref);
 
         auto file = store->toRealPath(
             downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
@@ -399,9 +399,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::string line;
         std::string id;
         while(getline(is, line)) {
-            auto index = line.find(ref_uri);
+            // Append $ to avoid partial name matches
-            if (index != std::string::npos) {
+            std::regex pattern(fmt("%s$", ref_uri));
-                id = line.substr(0, index-1);
+
+            if (std::regex_search(line, pattern)) {
+                id = line.substr(0, line.find('\t'));
                 break;
             }
         }

src/libstore/build/derivation-goal.cc

@@ -207,8 +207,6 @@ void DerivationGoal::haveDerivation()
     if (!drv->type().hasKnownOutputPaths())
         settings.requireExperimentalFeature(Xp::CaDerivations);
 
-    retrySubstitution = false;
-
     for (auto & i : drv->outputsAndOptPaths(worker.store))
         if (i.second.second)
             worker.store.addTempRoot(*i.second.second);
@@ -311,14 +309,11 @@ void DerivationGoal::outputsSubstitutionTried()
     gaveUpOnSubstitution();
 }
 
+
 /* At least one of the output paths could not be
    produced using a substitute.  So we have to build instead. */
 void DerivationGoal::gaveUpOnSubstitution()
 {
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
     /* The inputs must be built before we can build this goal. */
     if (useDerivation)
         for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
@@ -426,7 +421,8 @@ void DerivationGoal::inputsRealised()
         return;
     }
 
-    if (retrySubstitution) {
+    if (retrySubstitution && !retriedSubstitution) {
+        retriedSubstitution = true;
         haveDerivation();
         return;
     }

src/libstore/build/derivation-goal.hh

@@ -61,8 +61,12 @@ struct DerivationGoal : public Goal
     bool needRestart = false;
 
     /* Whether to retry substituting the outputs after building the
-       inputs. */
+       inputs. This is done in case of an incomplete closure. */
-    bool retrySubstitution;
+    bool retrySubstitution = false;
+
+    /* Whether we've retried substitution, in which case we won't try
+       again. */
+    bool retriedSubstitution = false;
 
     /* The derivation stored at drvPath. */
     std::unique_ptr<Derivation> drv;

src/libstore/build/goal.cc

@@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee)
 
 void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
 {
-    assert(waitees.find(waitee) != waitees.end());
+    assert(waitees.count(waitee));
     waitees.erase(waitee);
 
     trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));

src/libstore/build/goal.hh

@@ -40,21 +40,21 @@ struct Goal : public std::enable_shared_from_this<Goal>
     WeakGoals waiters;
 
     /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
+    size_t nrFailed = 0;
 
     /* Number of substitution goals we are/were waiting for that
        failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
+    size_t nrNoSubstituters = 0;
 
     /* Number of substitution goals we are/were waiting for that
        failed because they had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
+    size_t nrIncompleteClosure = 0;
 
     /* Name of this goal for debugging purposes. */
     std::string name;
 
     /* Whether the goal is finished. */
-    ExitCode exitCode;
+    ExitCode exitCode = ecBusy;
 
     /* Build result. */
     BuildResult buildResult;
@@ -65,10 +65,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
     Goal(Worker & worker, DerivedPath path)
         : worker(worker)
         , buildResult { .path = std::move(path) }
-    {
+    { }
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
-        exitCode = ecBusy;
-    }
 
     virtual ~Goal()
     {

src/libstore/build/local-derivation-goal.cc

@@ -2613,7 +2613,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);
         }
-        builtOutputs.emplace(thisRealisation.id, thisRealisation);
+        if (wantOutput(outputName, wantedOutputs))
+            builtOutputs.emplace(thisRealisation.id, thisRealisation);
     }
 
     return builtOutputs;

src/libstore/builtins/buildenv.cc

@@ -47,9 +47,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
             throw;
         }
 
-        /* The files below are special-cased to that they don't show up
+        /* The files below are special-cased to that they don't show
-         * in user profiles, either because they are useless, or
+         * up in user profiles, either because they are useless, or
-         * because they would cauase pointless collisions (e.g., each
+         * because they would cause pointless collisions (e.g., each
          * Python package brings its own
          * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
          */
@@ -57,7 +57,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
             hasSuffix(srcFile, "/nix-support") ||
             hasSuffix(srcFile, "/perllocal.pod") ||
             hasSuffix(srcFile, "/info/dir") ||
-            hasSuffix(srcFile, "/log"))
+            hasSuffix(srcFile, "/log") ||
+            hasSuffix(srcFile, "/manifest.nix") ||
+            hasSuffix(srcFile, "/manifest.json"))
             continue;
 
         else if (S_ISDIR(srcSt.st_mode)) {

src/libstore/derivations.cc

@@ -474,7 +474,7 @@ Sync<DrvHashes> drvHashes;
 /* Look up the derivation by value and memoize the
    `hashDerivationModulo` call.
  */
-static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
+static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
 {
     {
         auto hashes = drvHashes.lock();
@@ -509,7 +509,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
    don't leak the provenance of fixed outputs, reducing pointless cache
    misses as the build itself won't know this.
  */
-DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
+DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
 {
     auto type = drv.type();
 
@@ -524,7 +524,10 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
                 + store.printStorePath(dof.path(store, drv.name, i.first)));
             outputHashes.insert_or_assign(i.first, std::move(hash));
         }
-        return outputHashes;
+        return DrvHash{
+            .hashes = outputHashes,
+            .kind = DrvHash::Kind::Regular,
+        };
     }
 
     auto kind = std::visit(overloaded {
@@ -540,65 +543,36 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
         },
     }, drv.type().raw());
 
-    /* For other derivations, replace the inputs paths with recursive
-       calls to this function. */
     std::map<std::string, StringSet> inputs2;
     for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
         // Avoid lambda capture restriction with standard / Clang
         auto & inputOutputs = inputOutputs0;
         const auto & res = pathDerivationModulo(store, drvPath);
-        std::visit(overloaded {
+        if (res.kind == DrvHash::Kind::Deferred)
-            // Regular non-CA derivation, replace derivation
+            kind = DrvHash::Kind::Deferred;
-            [&](const DrvHash & drvHash) {
+        for (auto & outputName : inputOutputs) {
-                kind |= drvHash.kind;
+            const auto h = res.hashes.at(outputName);
-                inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
+            inputs2[h.to_string(Base16, false)].insert(outputName);
-            },
+        }
-            // CA derivation's output hashes
-            [&](const CaOutputHashes & outputHashes) {
-                std::set<std::string> justOut = { "out" };
-                for (auto & output : inputOutputs) {
-                    /* Put each one in with a single "out" output.. */
-                    const auto h = outputHashes.at(output);
-                    inputs2.insert_or_assign(
-                        h.to_string(Base16, false),
-                        justOut);
-                }
-            },
-        }, res.raw());
     }
 
     auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
 
-    return DrvHash { .hash = hash, .kind = kind };
+    std::map<std::string, Hash> outputHashes;
-}
+    for (const auto & [outputName, _] : drv.outputs) {
-
+        outputHashes.insert_or_assign(outputName, hash);
-
-void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
-{
-    switch (other) {
-    case DrvHash::Kind::Regular:
-        break;
-    case DrvHash::Kind::Deferred:
-        self = other;
-        break;
     }
+
+    return DrvHash {
+        .hashes = outputHashes,
+        .kind = kind,
+    };
 }
 
 
 std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
 {
-    std::map<std::string, Hash> res;
+    return hashDerivationModulo(store, drv, true).hashes;
-    std::visit(overloaded {
-        [&](const DrvHash & drvHash) {
-            for (auto & outputName : drv.outputNames()) {
-                res.insert({outputName, drvHash.hash});
-            }
-        },
-        [&](const CaOutputHashes & outputHashes) {
-            res = outputHashes;
-        },
-    }, hashDerivationModulo(store, drv, true).raw());
-    return res;
 }
 
 
@@ -747,7 +721,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
     auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
     for (auto & [outputName, output] : drv.outputs) {
         if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
-            auto & h = hashModulo.requireNoFixedNonDeferred();
+            auto & h = hashModulo.hashes.at(outputName);
             auto outPath = store.makeOutputPath(outputName, h, drv.name);
             drv.env[outputName] = store.printStorePath(outPath);
             output = DerivationOutput::InputAddressed {
@@ -758,13 +732,6 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
-    auto * drvHashOpt = std::get_if<DrvHash>(&raw());
-    assert(drvHashOpt);
-    assert(drvHashOpt->kind == DrvHash::Kind::Regular);
-    return drvHashOpt->hash;
-}
-
 static bool tryResolveInput(
     Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
     const StorePath & inputDrv, const StringSet & inputOutputs)

src/libstore/derivations.hh

@@ -202,12 +202,14 @@ bool isDerivation(const std::string & fileName);
    the output name is "out". */
 std::string outputPathName(std::string_view drvName, std::string_view outputName);
 
-// known CA drv's output hashes, current just for fixed-output derivations
-// whose output hashes are always known since they are fixed up-front.
-typedef std::map<std::string, Hash> CaOutputHashes;
 
+// The hashes modulo of a derivation.
+//
+// Each output is given a hash, although in practice only the content-addressed
+// derivations (fixed-output or not) will have a different hash for each
+// output.
 struct DrvHash {
-    Hash hash;
+    std::map<std::string, Hash> hashes;
 
     enum struct Kind: bool {
         // Statically determined derivations.
@@ -222,28 +224,6 @@ struct DrvHash {
 
 void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
 
-typedef std::variant<
-    // Regular normalized derivation hash, and whether it was deferred (because
-    // an ancestor derivation is a floating content addressed derivation).
-    DrvHash,
-    // Fixed-output derivation hashes
-    CaOutputHashes
-> _DrvHashModuloRaw;
-
-struct DrvHashModulo : _DrvHashModuloRaw {
-    using Raw = _DrvHashModuloRaw;
-    using Raw::Raw;
-
-    /* Get hash, throwing if it is per-output CA hashes or a
-       deferred Drv hash.
-     */
-    const Hash & requireNoFixedNonDeferred() const;
-
-    inline const Raw & raw() const {
-        return static_cast<const Raw &>(*this);
-    }
-};
-
 /* Returns hashes with the details of fixed-output subderivations
    expunged.
 
@@ -267,7 +247,7 @@ struct DrvHashModulo : _DrvHashModuloRaw {
    ATerm, after subderivations have been likewise expunged from that
    derivation.
  */
-DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
+DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
 
 /*
    Return a map associating each output to a hash that uniquely identifies its
@@ -276,7 +256,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
 std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
 
 /* Memoisation of hashDerivationModulo(). */
-typedef std::map<StorePath, DrvHashModulo> DrvHashes;
+typedef std::map<StorePath, DrvHash> DrvHashes;
 
 // FIXME: global, though at least thread-safe.
 extern Sync<DrvHashes> drvHashes;

src/libstore/derived-path.hh

@@ -25,6 +25,9 @@ struct DerivedPathOpaque {
     nlohmann::json toJSON(ref<Store> store) const;
     std::string to_string(const Store & store) const;
     static DerivedPathOpaque parse(const Store & store, std::string_view);
+
+    bool operator < (const DerivedPathOpaque & b) const
+    { return path < b.path; }
 };
 
 /**
@@ -46,6 +49,9 @@ struct DerivedPathBuilt {
     std::string to_string(const Store & store) const;
     static DerivedPathBuilt parse(const Store & store, std::string_view);
     nlohmann::json toJSON(ref<Store> store) const;
+
+    bool operator < (const DerivedPathBuilt & b) const
+    { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); }
 };
 
 using _DerivedPathRaw = std::variant<

src/libstore/local-store.cc

@@ -695,16 +695,15 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     // combinations that are currently prohibited.
     drv.type();
 
-    std::optional<Hash> h;
+    std::optional<DrvHash> hashesModulo;
     for (auto & i : drv.outputs) {
         std::visit(overloaded {
             [&](const DerivationOutput::InputAddressed & doia) {
-                if (!h) {
+                if (!hashesModulo) {
                     // somewhat expensive so we do lazily
-                    auto h0 = hashDerivationModulo(*this, drv, true);
+                    hashesModulo = hashDerivationModulo(*this, drv, true);
-                    h = h0.requireNoFixedNonDeferred();
                 }
-                StorePath recomputed = makeOutputPath(i.first, *h, drvName);
+                StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName);
                 if (doia.path != recomputed)
                     throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
                         printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));

src/libstore/make-content-addressed.cc

@@ -0,0 +1,80 @@
+#include "make-content-addressed.hh"
+#include "references.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths)
+{
+    StorePathSet closure;
+    srcStore.computeFSClosure(storePaths, closure);
+
+    auto paths = srcStore.topoSortPaths(closure);
+
+    std::reverse(paths.begin(), paths.end());
+
+    std::map<StorePath, StorePath> remappings;
+
+    for (auto & path : paths) {
+        auto pathS = srcStore.printStorePath(path);
+        auto oldInfo = srcStore.queryPathInfo(path);
+        std::string oldHashPart(path.hashPart());
+
+        StringSink sink;
+        srcStore.narFromPath(path, sink);
+
+        StringMap rewrites;
+
+        StorePathSet references;
+        bool hasSelfReference = false;
+        for (auto & ref : oldInfo->references) {
+            if (ref == path)
+                hasSelfReference = true;
+            else {
+                auto i = remappings.find(ref);
+                auto replacement = i != remappings.end() ? i->second : ref;
+                // FIXME: warn about unremapped paths?
+                if (replacement != ref)
+                    rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
+                references.insert(std::move(replacement));
+            }
+        }
+
+        sink.s = rewriteStrings(sink.s, rewrites);
+
+        HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+        hashModuloSink(sink.s);
+
+        auto narModuloHash = hashModuloSink.finish().first;
+
+        auto dstPath = dstStore.makeFixedOutputPath(
+            FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference);
+
+        printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath));
+
+        StringSink sink2;
+        RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2);
+        rsink2(sink.s);
+        rsink2.flush();
+
+        ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) };
+        info.references = std::move(references);
+        if (hasSelfReference) info.references.insert(info.path);
+        info.narSize = sink.s.size();
+        info.ca = FixedOutputHash {
+            .method = FileIngestionMethod::Recursive,
+            .hash = narModuloHash,
+        };
+
+        StringSource source(sink2.s);
+        dstStore.addToStore(info, source);
+
+        remappings.insert_or_assign(std::move(path), std::move(info.path));
+    }
+
+    return remappings;
+}
+
+}

src/libstore/make-content-addressed.hh

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths);
+
+}

src/libutil/experimental-features.cc

@@ -11,6 +11,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::NixCommand, "nix-command" },
     { Xp::RecursiveNix, "recursive-nix" },
     { Xp::NoUrlLiterals, "no-url-literals" },
+    { Xp::FetchClosure, "fetch-closure" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)

src/libutil/experimental-features.hh

@@ -19,7 +19,8 @@ enum struct ExperimentalFeature
     Flakes,
     NixCommand,
     RecursiveNix,
-    NoUrlLiterals
+    NoUrlLiterals,
+    FetchClosure,
 };
 
 /**

src/libutil/tarfile.cc

@@ -39,30 +39,32 @@ void TarArchive::check(int err, const std::string & reason)
         throw Error(reason, archive_error_string(this->archive));
 }
 
-TarArchive::TarArchive(Source & source, bool raw)
+TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
-    : source(&source), buffer(4096)
 {
-    init();
+    this->archive = archive_read_new();
-    if (!raw)
+    this->source = &source;
+
+    if (!raw) {
+        archive_read_support_filter_all(archive);
         archive_read_support_format_all(archive);
-    else
+    } else {
+        archive_read_support_filter_all(archive);
         archive_read_support_format_raw(archive);
+        archive_read_support_format_empty(archive);
+    }
     check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
 }
 
+
 TarArchive::TarArchive(const Path & path)
 {
-    init();
+    this->archive = archive_read_new();
+
+    archive_read_support_filter_all(archive);
     archive_read_support_format_all(archive);
     check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
 }
 
-void TarArchive::init()
-{
-    archive = archive_read_new();
-    archive_read_support_filter_all(archive);
-}
-
 void TarArchive::close()
 {
     check(archive_read_close(this->archive), "Failed to close archive (%s)");

src/libutil/tarfile.hh

@@ -17,13 +17,10 @@ struct TarArchive {
     // disable copy constructor
     TarArchive(const TarArchive &) = delete;
 
-    void init();
-
     void close();
 
     ~TarArchive();
 };
-
 void unpackTarfile(Source & source, const Path & destDir);
 
 void unpackTarfile(const Path & tarFile, const Path & destDir);

src/nix-env/user-env.cc

@@ -105,8 +105,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     /* Also write a copy of the list of user environment elements to
        the store; we need it for future modifications of the
        environment. */
+    std::ostringstream str;
+    manifest.print(str, true);
     auto manifestFile = state.store->addTextToStore("env-manifest.nix",
-        fmt("%s", manifest), references);
+        str.str(), references);
 
     /* Get the environment builder expression. */
     Value envBuilder;

src/nix/app.cc

@@ -71,7 +71,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
 
         std::vector<StorePathWithOutputs> context2;
         for (auto & [path, name] : context)
-            context2.push_back({state.store->parseStorePath(path), {name}});
+            context2.push_back({path, {name}});
 
         return UnresolvedApp{App {
             .context = std::move(context2),

src/nix/develop.cc

@@ -204,10 +204,10 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
             output.second = DerivationOutput::Deferred { };
             drv.env[output.first] = "";
         }
-        auto h0 = hashDerivationModulo(*evalStore, drv, true);
+        auto hashesModulo = hashDerivationModulo(*evalStore, drv, true);
-        const Hash & h = h0.requireNoFixedNonDeferred();
 
         for (auto & output : drv.outputs) {
+            Hash h = hashesModulo.hashes.at(output.first);
             auto outPath = store->makeOutputPath(output.first, h, drv.name);
             output.second = DerivationOutput::InputAddressed {
                 .path = outPath,

src/nix/edit.md

@@ -24,8 +24,8 @@ this attribute to the location of the definition of the
 `meta.description`, `version` or `name` derivation attributes.
 
 The editor to invoke is specified by the `EDITOR` environment
-variable. It defaults to `cat`. If the editor is `emacs`, `nano` or
+variable. It defaults to `cat`. If the editor is `emacs`, `nano`,
-`vim`, it is passed the line number of the derivation using the
+`vim` or `kak`, it is passed the line number of the derivation using
-argument `+<lineno>`.
+the argument `+<lineno>`.
 
 )""

src/nix/main.cc

@@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         {"hash-path", {"hash", "path"}},
         {"ls-nar", {"nar", "ls"}},
         {"ls-store", {"store", "ls"}},
-        {"make-content-addressable", {"store", "make-content-addressable"}},
+        {"make-content-addressable", {"store", "make-content-addressed"}},
         {"optimise-store", {"store", "optimise"}},
         {"ping-store", {"store", "ping"}},
         {"sign-paths", {"store", "sign"}},
@@ -289,6 +289,7 @@ void mainWrapped(int argc, char * * argv)
     }
 
     if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
+        settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure};
         evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"));
         auto res = nlohmann::json::object();

src/nix/make-content-addressable.cc

@@ -1,102 +0,0 @@
-#include "command.hh"
-#include "store-api.hh"
-#include "references.hh"
-#include "common-args.hh"
-#include "json.hh"
-
-using namespace nix;
-
-struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
-{
-    CmdMakeContentAddressable()
-    {
-        realiseMode = Realise::Outputs;
-    }
-
-    std::string description() override
-    {
-        return "rewrite a path or closure to content-addressed form";
-    }
-
-    std::string doc() override
-    {
-        return
-          #include "make-content-addressable.md"
-          ;
-    }
-
-    void run(ref<Store> store, StorePaths && storePaths) override
-    {
-        auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
-
-        std::reverse(paths.begin(), paths.end());
-
-        std::map<StorePath, StorePath> remappings;
-
-        auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
-        auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
-
-        for (auto & path : paths) {
-            auto pathS = store->printStorePath(path);
-            auto oldInfo = store->queryPathInfo(path);
-            std::string oldHashPart(path.hashPart());
-
-            StringSink sink;
-            store->narFromPath(path, sink);
-
-            StringMap rewrites;
-
-            StorePathSet references;
-            bool hasSelfReference = false;
-            for (auto & ref : oldInfo->references) {
-                if (ref == path)
-                    hasSelfReference = true;
-                else {
-                    auto i = remappings.find(ref);
-                    auto replacement = i != remappings.end() ? i->second : ref;
-                    // FIXME: warn about unremapped paths?
-                    if (replacement != ref)
-                        rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
-                    references.insert(std::move(replacement));
-                }
-            }
-
-            sink.s = rewriteStrings(sink.s, rewrites);
-
-            HashModuloSink hashModuloSink(htSHA256, oldHashPart);
-            hashModuloSink(sink.s);
-
-            auto narHash = hashModuloSink.finish().first;
-
-            ValidPathInfo info {
-                store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
-                narHash,
-            };
-            info.references = std::move(references);
-            if (hasSelfReference) info.references.insert(info.path);
-            info.narSize = sink.s.size();
-            info.ca = FixedOutputHash {
-                .method = FileIngestionMethod::Recursive,
-                .hash = info.narHash,
-            };
-
-            if (!json)
-                notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
-
-            auto source = sinkToSource([&](Sink & nextSink) {
-                RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
-                rsink2(sink.s);
-                rsink2.flush();
-            });
-
-            store->addToStore(info, *source);
-
-            if (json)
-                jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
-
-            remappings.insert_or_assign(std::move(path), std::move(info.path));
-        }
-    }
-};
-
-static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});

src/nix/make-content-addressed.cc

@@ -0,0 +1,55 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "common-args.hh"
+#include "json.hh"
+
+using namespace nix;
+
+struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
+{
+    CmdMakeContentAddressed()
+    {
+        realiseMode = Realise::Outputs;
+    }
+
+    std::string description() override
+    {
+        return "rewrite a path or closure to content-addressed form";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "make-content-addressed.md"
+          ;
+    }
+
+    void run(ref<Store> srcStore, StorePaths && storePaths) override
+    {
+        auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
+
+        auto remappings = makeContentAddressed(*srcStore, *dstStore,
+            StorePathSet(storePaths.begin(), storePaths.end()));
+
+        if (json) {
+            JSONObject jsonRoot(std::cout);
+            JSONObject jsonRewrites(jsonRoot.object("rewrites"));
+            for (auto & path : storePaths) {
+                auto i = remappings.find(path);
+                assert(i != remappings.end());
+                jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second));
+            }
+        } else {
+            for (auto & path : storePaths) {
+                auto i = remappings.find(path);
+                assert(i != remappings.end());
+                notice("rewrote '%s' to '%s'",
+                    srcStore->printStorePath(path),
+                    srcStore->printStorePath(i->second));
+            }
+        }
+    }
+};
+
+static auto rCmdMakeContentAddressed = registerCommand2<CmdMakeContentAddressed>({"store", "make-content-addressed"});

src/nix/make-content-addressed.md

@@ -5,7 +5,7 @@ R""(
 * Create a content-addressed representation of the closure of GNU Hello:
 
   ```console
-  # nix store make-content-addressable -r nixpkgs#hello
+  # nix store make-content-addressed nixpkgs#hello
   …
   rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
   ```
@@ -29,7 +29,7 @@ R""(
   system closure:
 
   ```console
-  # nix store make-content-addressable -r /run/current-system
+  # nix store make-content-addressed /run/current-system
   ```
 
 # Description

src/nix/profile.cc

@@ -62,22 +62,21 @@ struct ProfileElement
         return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
     }
 
-    void updateStorePaths(ref<Store> evalStore, ref<Store> store, Installable & installable)
+    void updateStorePaths(
+        ref<Store> evalStore,
+        ref<Store> store,
+        const BuiltPaths & builtPaths)
     {
         // FIXME: respect meta.outputsToInstall
         storePaths.clear();
-        for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) {
+        for (auto & buildable : builtPaths) {
             std::visit(overloaded {
                 [&](const BuiltPath::Opaque & bo) {
                     storePaths.insert(bo.path);
                 },
                 [&](const BuiltPath::Built & bfd) {
-                    // TODO: Why are we querying if we know the output
+                    for (auto & output : bfd.outputs)
-                    // names already? Is it just to figure out what the
-                    // default one is?
-                    for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
                         storePaths.insert(output.second);
-                    }
                 },
             }, buildable.raw());
         }
@@ -107,8 +106,9 @@ struct ProfileManifest
                     element.storePaths.insert(state.store->parseStorePath((std::string) p));
                 element.active = e["active"];
                 if (e.value("uri", "") != "") {
+                    auto originalUrl = e.value("originalUrl", e["originalUri"]);
                     element.source = ProfileElementSource{
-                        parseFlakeRef(e["originalUri"]),
+                        parseFlakeRef(originalUrl),
                         parseFlakeRef(e["uri"]),
                         e["attrPath"]
                     };
@@ -143,7 +143,7 @@ struct ProfileManifest
             obj["storePaths"] = paths;
             obj["active"] = element.active;
             if (element.source) {
-                obj["originalUri"] = element.source->originalRef.to_string();
+                obj["originalUrl"] = element.source->originalRef.to_string();
                 obj["uri"] = element.source->resolvedRef.to_string();
                 obj["attrPath"] = element.source->attrPath;
             }
@@ -235,6 +235,16 @@ struct ProfileManifest
     }
 };
 
+static std::map<Installable *, BuiltPaths>
+builtPathsPerInstallable(
+    const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths)
+{
+    std::map<Installable *, BuiltPaths> res;
+    for (auto & [installable, builtPath] : builtPaths)
+        res[installable.get()].push_back(builtPath);
+    return res;
+}
+
 struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 {
     std::string description() override
@@ -253,7 +263,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
     {
         ProfileManifest manifest(*getEvalState(), *profile);
 
-        auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal);
+        auto builtPaths = builtPathsPerInstallable(
+            Installable::build2(
+                getEvalStore(), store, Realise::Outputs, installables, bmNormal));
 
         for (auto & installable : installables) {
             ProfileElement element;
@@ -268,7 +280,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                 };
             }
 
-            element.updateStorePaths(getEvalStore(), store, *installable);
+            element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
 
             manifest.elements.push_back(std::move(element));
         }
@@ -456,12 +468,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
             warn ("Use 'nix profile list' to see the current profile.");
         }
 
-        auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal);
+        auto builtPaths = builtPathsPerInstallable(
+            Installable::build2(
+                getEvalStore(), store, Realise::Outputs, installables, bmNormal));
 
         for (size_t i = 0; i < installables.size(); ++i) {
             auto & installable = installables.at(i);
             auto & element = manifest.elements[indices.at(i)];
-            element.updateStorePaths(getEvalStore(), store, *installable);
+            element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
         }
 
         updateProfile(manifest.build(store));

src/nix/profile.md

@@ -70,7 +70,7 @@ are installed in this version of the profile. It looks like this:
     {
       "active": true,
       "attrPath": "legacyPackages.x86_64-linux.zoom-us",
-      "originalUri": "flake:nixpkgs",
+      "originalUrl": "flake:nixpkgs",
       "storePaths": [
         "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
       ],
@@ -84,11 +84,11 @@ are installed in this version of the profile. It looks like this:
 Each object in the array `elements` denotes an installed package and
 has the following fields:
 
-* `originalUri`: The [flake reference](./nix3-flake.md) specified by
+* `originalUrl`: The [flake reference](./nix3-flake.md) specified by
   the user at the time of installation (e.g. `nixpkgs`). This is also
   the flake reference that will be used by `nix profile upgrade`.
 
-* `uri`: The immutable flake reference to which `originalUri`
+* `uri`: The immutable flake reference to which `originalUrl`
   resolved.
 
 * `attrPath`: The flake output attribute that provided this

src/nix/repl.cc

@@ -396,6 +396,7 @@ StorePath NixRepl::getDerivationPath(Value & v) {
 
 bool NixRepl::processLine(std::string line)
 {
+    line = trim(line);
     if (line == "") return true;
 
     _isInterrupted = false;

src/nix/run.cc

@@ -38,9 +38,12 @@ void runProgramInStore(ref<Store> store,
        unshare(CLONE_NEWUSER) doesn't work in a multithreaded program
        (which "nix" is), so we exec() a single-threaded helper program
        (chrootHelper() below) to do the work. */
-    auto store2 = store.dynamic_pointer_cast<LocalStore>();
+    auto store2 = store.dynamic_pointer_cast<LocalFSStore>();
 
-    if (store2 && store->storeDir != store2->getRealStoreDir()) {
+    if (!store2)
+        throw Error("store '%s' is not a local store so it does not support command execution", store->getUri());
+
+    if (store->storeDir != store2->getRealStoreDir()) {
         Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
         for (auto & arg : args) helperArgs.push_back(arg);
 

tests/binary-cache.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-needLocalStore "“--no-require-sigs” can’t be used with the daemon"
+needLocalStore "'--no-require-sigs' can’t be used with the daemon"
 
 # We can produce drvs directly into the binary cache
 clearStore

tests/build.sh

@@ -1,15 +1,27 @@
 source common.sh
 
-expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
+clearStore
+
+# Make sure that 'nix build' only returns the outputs we asked for.
+nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys | length == 1) and
+    (.outputs.first | match(".*multiple-outputs-a-first")))
+'
+
 nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys | length == 2) and
     (.outputs.first | match(".*multiple-outputs-a-first")) and
     (.outputs.second | match(".*multiple-outputs-a-second")))
   and (.[1] |
     (.drvPath | match(".*multiple-outputs-b.drv")) and
+    (.outputs | keys | length == 1) and
     (.outputs.out | match(".*multiple-outputs-b")))
 '
+
 testNormalization () {
     clearStore
     outPath=$(nix-build ./simple.nix --no-out-link)

tests/fetchClosure.sh

@@ -0,0 +1,58 @@
+source common.sh
+
+enableFeatures "fetch-closure"
+needLocalStore "'--no-require-sigs' can’t be used with the daemon"
+
+clearStore
+clearCacheCache
+
+# Initialize binary cache.
+nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out)
+caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')
+nix copy --to file://$cacheDir $nonCaPath
+
+# Test basic fetchClosure rewriting from non-CA to CA.
+clearStore
+
+[ ! -e $nonCaPath ]
+[ ! -e $caPath ]
+
+[[ $(nix eval -v --raw --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $nonCaPath;
+    toPath = $caPath;
+  }
+") = $caPath ]]
+
+[ ! -e $nonCaPath ]
+[ -e $caPath ]
+
+# In impure mode, we can use non-CA paths.
+[[ $(nix eval --raw --no-require-sigs --impure --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $nonCaPath;
+  }
+") = $nonCaPath ]]
+
+[ -e $nonCaPath ]
+
+# 'toPath' set to empty string should fail but print the expected path.
+nix eval -v --json --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $nonCaPath;
+    toPath = \"\";
+  }
+" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath"
+
+# If fromPath is CA, then toPath isn't needed.
+nix copy --to file://$cacheDir $caPath
+
+[[ $(nix eval -v --raw --expr "
+  builtins.fetchClosure {
+    fromStore = \"file://$cacheDir\";
+    fromPath = $caPath;
+  }
+") = $caPath ]]

tests/fetchPath.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-touch foo -t 202211111111
+touch $TEST_ROOT/foo -t 202211111111
 # We only check whether 2022-11-1* **:**:** is the last modified date since
 # `lastModified` is transformed into UTC in `builtins.fetchTarball`.
-[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2022111.* ]]
+[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]]

tests/local.mk

@@ -98,7 +98,8 @@ nix_tests = \
   flake-print-local-path-error.sh \
   nix-profile.sh \
   suggestions.sh \
-  store-ping.sh
+  store-ping.sh \
+  fetchClosure.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	nix_tests += compute-levels.sh

tests/logging.sh

@@ -13,3 +13,14 @@ rm -rf $NIX_LOG_DIR
 (! nix-store -l $path)
 nix-build dependencies.nix --no-out-link --compress-build-log
 [ "$(nix-store -l $path)" = FOO ]
+
+# test whether empty logs work fine with `nix log`.
+builder="$(mktemp)"
+echo -e "#!/bin/sh\nmkdir \$out" > "$builder"
+outp="$(nix-build -E \
+    'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \
+    --out-link "$(mktemp -d)/result")"
+
+test -d "$outp"
+
+nix log "$outp"

tests/sourcehut-flakes.nix

@@ -59,7 +59,7 @@ let
       echo 'ref: refs/heads/master' > $out/HEAD
 
       mkdir -p $out/info
-      echo '${nixpkgs.rev} refs/heads/master' > $out/info/refs
+      echo -e '${nixpkgs.rev}\trefs/heads/master' > $out/info/refs
     '';
 
 in

tests/user-envs.nix

@@ -8,6 +8,8 @@ assert foo == "foo";
 
 let
 
+  platforms = let x = "foobar"; in [ x x ];
+
   makeDrv = name: progName: (mkDerivation {
     name = assert progName != "fail"; name;
     inherit progName system;
@@ -15,6 +17,7 @@ let
   } // {
     meta = {
       description = "A silly test package with some \${escaped anti-quotation} in it";
+      inherit platforms;
     };
   });