Merge remote-tracking branch 'origin/master' into debug-exploratory-PR

doc/manual/src/release-notes/rl-next.md

@@ -14,3 +14,13 @@
 
 * `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
   This matches the default behaviour of `nix-build`.
+
+* You can now specify which outputs of a derivation `nix` should
+  operate on using the syntax `installable^outputs`,
+  e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
+  `nix` will use the outputs specified by the derivation's
+  `meta.outputsToInstall` attribute if it exists, or all outputs
+  otherwise.
+
+  Selecting derivation outputs using the attribute selection syntax
+  (e.g. `nixpkgs#glibc.dev`) no longer works.

src/libcmd/installables.cc

@@ -440,10 +440,8 @@ DerivedPaths InstallableValue::toDerivedPaths()
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
-        auto outputName = drv.outputName;
-        if (outputName == "")
-            throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
-        drvsToOutputs[drv.drvPath].insert(outputName);
+        for (auto & outputName : drv.outputsToInstall)
+            drvsToOutputs[drv.drvPath].insert(outputName);
         drvsToCopy.insert(drv.drvPath);
     }
 
@@ -466,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
     SourceExprCommand & cmd;
     RootValue v;
     std::string attrPath;
+    OutputsSpec outputsSpec;
 
-    InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath)
-        : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
+    InstallableAttrPath(
+        ref<EvalState> state,
+        SourceExprCommand & cmd,
+        Value * v,
+        const std::string & attrPath,
+        OutputsSpec outputsSpec)
+        : InstallableValue(state)
+        , cmd(cmd)
+        , v(allocRootValue(v))
+        , attrPath(attrPath)
+        , outputsSpec(std::move(outputsSpec))
     { }
 
     std::string what() const override { return attrPath; }
@@ -497,7 +505,19 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
         auto drvPath = drvInfo.queryDrvPath();
         if (!drvPath)
             throw Error("'%s' is not a derivation", what());
-        res.push_back({ *drvPath, drvInfo.queryOutputName() });
+
+        std::set<std::string> outputsToInstall;
+
+        if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+            outputsToInstall = *outputNames;
+        else
+            for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
+                outputsToInstall.insert(output.first);
+
+        res.push_back(DerivationInfo {
+            .drvPath = *drvPath,
+            .outputsToInstall = std::move(outputsToInstall)
+        });
     }
 
     return res;
@@ -574,6 +594,7 @@ InstallableFlake::InstallableFlake(
     ref<EvalState> state,
     FlakeRef && flakeRef,
     std::string_view fragment,
+    OutputsSpec outputsSpec,
     Strings attrPaths,
     Strings prefixes,
     const flake::LockFlags & lockFlags)
@@ -581,6 +602,7 @@ InstallableFlake::InstallableFlake(
       flakeRef(flakeRef),
       attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
       prefixes(fragment == "" ? Strings{} : prefixes),
+      outputsSpec(std::move(outputsSpec)),
       lockFlags(lockFlags)
 {
     if (cmd && cmd->getAutoArgs(*state)->size())
@@ -598,9 +620,29 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
 
     auto drvPath = attr->forceDerivation();
 
+    std::set<std::string> outputsToInstall;
+
+    if (auto aMeta = attr->maybeGetAttr(state->sMeta))
+        if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
+            for (auto & s : aOutputsToInstall->getListOfStrings())
+                outputsToInstall.insert(s);
+
+    if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
+        outputsToInstall.clear();
+        if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
+            for (auto & s : aOutputs->getListOfStrings())
+                outputsToInstall.insert(s);
+    }
+
+    if (outputsToInstall.empty())
+        outputsToInstall.insert("out");
+
+    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        outputsToInstall = *outputNames;
+
     auto drvInfo = DerivationInfo {
-        std::move(drvPath),
-        attr->getAttr(state->sOutputName)->getString()
+        .drvPath = std::move(drvPath),
+        .outputsToInstall = std::move(outputsToInstall),
     };
 
     return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};
@@ -723,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             state->eval(e, *vFile);
         }
 
-        for (auto & s : ss)
-            result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s));
+        for (auto & s : ss) {
+            auto [prefix, outputsSpec] = parseOutputsSpec(s);
+            result.push_back(
+                std::make_shared<InstallableAttrPath>(
+                    state, *this, vFile,
+                    prefix == "." ? "" : prefix,
+                    outputsSpec));
+        }
 
     } else {
 
@@ -743,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
+                auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
                         this,
                         getEvalState(),
                         std::move(flakeRef),
                         fragment,
+                        outputsSpec,
                         getDefaultFlakeAttrPaths(),
                         getDefaultFlakeAttrPathPrefixes(),
                         lockFlags));
@@ -822,12 +871,13 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                         auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
                         auto drvOutputs = drv.outputsAndOptPaths(*store);
                         for (auto & output : bfd.outputs) {
-                            if (!outputHashes.count(output))
+                            auto outputHash = get(outputHashes, output);
+                            if (!outputHash)
                                 throw Error(
                                     "the derivation '%s' doesn't have an output named '%s'",
                                     store->printStorePath(bfd.drvPath), output);
                             if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
-                                DrvOutput outputId { outputHashes.at(output), output };
+                                DrvOutput outputId { *outputHash, output };
                                 auto realisation = store->queryRealisation(outputId);
                                 if (!realisation)
                                     throw Error(
@@ -838,10 +888,11 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                             } else {
                                 // If ca-derivations isn't enabled, assume that
                                 // the output path is statically known.
-                                assert(drvOutputs.count(output));
-                                assert(drvOutputs.at(output).second);
+                                auto drvOutput = get(drvOutputs, output);
+                                assert(drvOutput);
+                                assert(drvOutput->second);
                                 outputs.insert_or_assign(
-                                    output, *drvOutputs.at(output).second);
+                                    output, *drvOutput->second);
                             }
                         }
                         res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});

src/libcmd/installables.hh

@@ -141,7 +141,7 @@ struct InstallableValue : Installable
     struct DerivationInfo
     {
         StorePath drvPath;
-        std::string outputName;
+        std::set<std::string> outputsToInstall;
     };
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;
@@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
     FlakeRef flakeRef;
     Strings attrPaths;
     Strings prefixes;
+    OutputsSpec outputsSpec;
     const flake::LockFlags & lockFlags;
     mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
@@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
         ref<EvalState> state,
         FlakeRef && flakeRef,
         std::string_view fragment,
+        OutputsSpec outputsSpec,
         Strings attrPaths,
         Strings prefixes,
         const flake::LockFlags & lockFlags);

src/libexpr/eval-cache.cc

@@ -47,7 +47,7 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
         createDirs(cacheDir);
 
         Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@@ -175,6 +175,24 @@ struct AttrDb
         });
     }
 
+    AttrId setListOfStrings(
+        AttrKey key,
+        const std::vector<std::string> & l)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (symbols[key.second])
+                (AttrType::ListOfStrings)
+                (concatStringsSep("\t", l)).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
     AttrId setPlaceholder(AttrKey key)
     {
         return doSQLite([&]()
@@ -269,6 +287,8 @@ struct AttrDb
             }
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::ListOfStrings:
+                return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
             case AttrType::Missing:
                 return {{rowId, missing_t()}};
             case AttrType::Misc:
@@ -385,7 +405,7 @@ std::string AttrCursor::getAttrPathStr(Symbol name) const
 
 Value & AttrCursor::forceValue()
 {
-    debug("evaluating uncached attribute %s", getAttrPathStr());
+    debug("evaluating uncached attribute '%s'", getAttrPathStr());
 
     auto & v = getValue();
 
@@ -626,6 +646,38 @@ bool AttrCursor::getBool()
     return v.boolean;
 }
 
+std::vector<std::string> AttrCursor::getListOfStrings()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey());
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto l = std::get_if<std::vector<std::string>>(&cachedValue->second)) {
+                debug("using cached list of strings attribute '%s'", getAttrPathStr());
+                return *l;
+            } else
+                throw TypeError("'%s' is not a list of strings", getAttrPathStr());
+        }
+    }
+
+    debug("evaluating uncached attribute '%s'", getAttrPathStr());
+
+    auto & v = getValue();
+    root->state.forceValue(v, noPos);
+
+    if (v.type() != nList)
+        throw TypeError("'%s' is not a list", getAttrPathStr());
+
+    std::vector<std::string> res;
+
+    for (auto & elem : v.listItems())
+        res.push_back(std::string(root->state.forceStringNoCtx(*elem)));
+
+    cachedValue = {root->db->setListOfStrings(getKey(), res), res};
+
+    return res;
+}
+
 std::vector<Symbol> AttrCursor::getAttrs()
 {
     if (root->db) {

src/libexpr/eval-cache.hh

@@ -44,6 +44,7 @@ enum AttrType {
     Misc = 4,
     Failed = 5,
     Bool = 6,
+    ListOfStrings = 7,
 };
 
 struct placeholder_t {};
@@ -61,7 +62,8 @@ typedef std::variant<
     missing_t,
     misc_t,
     failed_t,
-    bool
+    bool,
+    std::vector<std::string>
     > AttrValue;
 
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
@@ -114,6 +116,8 @@ public:
 
     bool getBool();
 
+    std::vector<std::string> getListOfStrings();
+
     std::vector<Symbol> getAttrs();
 
     bool isDerivation();

src/libexpr/flake/config.cc

@@ -50,13 +50,11 @@ void ConfigFile::apply()
         else
             assert(false);
 
-        if (!whitelist.count(baseName)) {
-            auto trustedList = readTrustedList();
-
+        if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) {
             bool trusted = false;
-            if (nix::fetchSettings.acceptFlakeConfig){
-                trusted = true;
-            } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
+            auto trustedList = readTrustedList();
+            auto tlname = get(trustedList, name);
+            if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
                 trusted = *saved;
                 warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
             } else {
@@ -69,7 +67,6 @@ void ConfigFile::apply()
                     writeTrustedList(trustedList);
                 }
             }
-
             if (!trusted) {
                 warn("ignoring untrusted flake configuration setting '%s'", name);
                 continue;

src/libexpr/flake/flakeref.cc

@@ -176,7 +176,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
                             parsedURL.query.insert_or_assign("shallow", "1");
 
                         return std::make_pair(
-                            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                            FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
                             fragment);
                     }
 
@@ -189,7 +189,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
             if (!hasPrefix(path, "/"))
                 throw BadURL("flake reference '%s' is not an absolute path", url);
             auto query = decodeQuery(match[2]);
-            path = canonPath(path + "/" + get(query, "dir").value_or(""));
+            path = canonPath(path + "/" + getOr(query, "dir", ""));
         }
 
         fetchers::Attrs attrs;
@@ -208,7 +208,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         input.parent = baseDir;
 
         return std::make_pair(
-            FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")),
+            FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
             fragment);
     }
 }
@@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
     return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
 }
 
+std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
+{
+    auto [prefix, outputsSpec] = parseOutputsSpec(url);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
+    return {std::move(flakeRef), fragment, outputsSpec};
+}
+
 }

src/libexpr/flake/flakeref.hh

@@ -3,6 +3,7 @@
 #include "types.hh"
 #include "hash.hh"
 #include "fetchers.hh"
+#include "path-with-outputs.hh"
 
 #include <variant>
 
@@ -79,4 +80,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
+std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
+
+
 }

src/libexpr/get-drvs.cc

@@ -34,7 +34,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 
     outputName =
         selectedOutputs.empty()
-        ? get(drv.env, "outputName").value_or("out")
+        ? getOr(drv.env, "outputName", "out")
         : *selectedOutputs.begin();
 
     auto i = drv.outputs.find(outputName);

src/libexpr/primops.cc

@@ -76,18 +76,18 @@ StringMap EvalState::realiseContext(const PathSet & context)
 
     /* Get all the output paths corresponding to the placeholders we had */
     for (auto & [drvPath, outputs] : drvs) {
-        auto outputPaths = store->queryDerivationOutputMap(drvPath);
+        const auto outputPaths = store->queryDerivationOutputMap(drvPath);
         for (auto & outputName : outputs) {
-            if (outputPaths.count(outputName) == 0)
-            {
+            auto outputPath = get(outputPaths, outputName);
+            if (!outputPath) {
                 auto e = Error("derivation '%s' does not have an output named '%s'",
-                        store->printStorePath(drvPath), outputName);
+                    store->printStorePath(drvPath), outputName);
                 debugLastTrace(e);
                 throw e;
             }
             res.insert_or_assign(
                 downstreamPlaceholder(*store, drvPath, outputName),
-                store->printStorePath(outputPaths.at(outputName))
+                store->printStorePath(*outputPath)
             );
         }
     }
@@ -1369,8 +1369,13 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
         switch (hashModulo.kind) {
         case DrvHash::Kind::Regular:
             for (auto & i : outputs) {
-                auto h = hashModulo.hashes.at(i);
-                auto outPath = state.store->makeOutputPath(i, h, drvName);
+                auto h = get(hashModulo.hashes, i);
+                if (!h)
+                    throw AssertionError({
+                        .msg = hintfmt("derivation produced no hash for output '%s'", i),
+                        .errPos = state.positions[posDrvName],
+                    });
+                auto outPath = state.store->makeOutputPath(i, *h, drvName);
                 drv.env[i] = state.store->printStorePath(outPath);
                 drv.outputs.insert_or_assign(
                     i,

src/libfetchers/git-utils.cc

@@ -0,0 +1,27 @@
+#include "git-utils.hh"
+
+#include <regex>
+
+std::optional<std::string> parseListReferenceHeadRef(std::string_view line)
+{
+    const static std::regex head_ref_regex("^ref: ([^\\s]+)\\t+HEAD$");
+    std::match_results<std::string_view::const_iterator> match;
+    if (std::regex_match(line.cbegin(), line.cend(), match, head_ref_regex)) {
+        return match[1];
+    } else {
+        return std::nullopt;
+    }
+}
+
+std::optional<std::string> parseListReferenceForRev(std::string_view rev, std::string_view line)
+{
+    const static std::regex rev_regex("^([^\\t]+)\\t+(.*)$");
+    std::match_results<std::string_view::const_iterator> match;
+    if (!std::regex_match(line.cbegin(), line.cend(), match, rev_regex)) {
+        return std::nullopt;
+    }
+    if (rev != match[2].str()) {
+        return std::nullopt;
+    }
+    return match[1];
+}

src/libfetchers/git-utils.hh

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <optional>
+
+// Parses the HEAD ref as reported by `git ls-remote --symref`
+//
+// Returns the head branch name as reported by `git ls-remote --symref`, e.g., if
+// ls-remote returns the output below, "main" is returned based on the ref line.
+//
+//   ref: refs/heads/main       HEAD
+//
+// If the repository is in 'detached head' state (HEAD is pointing to a rev
+// instead of a branch), parseListReferenceForRev("HEAD") may be used instead.
+std::optional<std::string> parseListReferenceHeadRef(std::string_view line);
+
+// Parses a reference line from `git ls-remote --symref`, e.g.,
+// parseListReferenceForRev("refs/heads/master", line) will return 6926...
+// given the line below.
+//
+// 6926beab444c33fb57b21819b6642d032016bb1e	refs/heads/master
+std::optional<std::string> parseListReferenceForRev(std::string_view rev, std::string_view line);

src/libfetchers/git.cc

@@ -5,15 +5,20 @@
 #include "store-api.hh"
 #include "url-parts.hh"
 #include "pathlocks.hh"
+#include "util.hh"
+#include "git-utils.hh"
 
 #include "fetch-settings.hh"
 
+#include <regex>
+#include <string.h>
 #include <sys/time.h>
 #include <sys/wait.h>
 
 using namespace std::string_literals;
 
 namespace nix::fetchers {
+namespace {
 
 // Explicit initial branch of our bare repo to suppress warnings from new version of git.
 // The value itself does not matter, since we always fetch a specific revision or branch.
@@ -21,25 +26,228 @@ namespace nix::fetchers {
 // old version of git, which will ignore unrecognized `-c` options.
 const std::string gitInitialBranch = "__nix_dummy_branch";
 
-static std::string getGitDir()
+std::string getGitDir()
 {
-    auto gitDir = getEnv("GIT_DIR");
-    if (!gitDir) {
-        return ".git";
+    return getEnv("GIT_DIR").value_or(".git");
+}
+
+bool isCacheFileWithinTtl(const time_t now, const struct stat & st)
+{
+    return st.st_mtime + settings.tarballTtl > now;
+}
+
+bool touchCacheFile(const Path& path, const time_t& touch_time)
+{
+  struct timeval times[2];
+  times[0].tv_sec = touch_time;
+  times[0].tv_usec = 0;
+  times[1].tv_sec = touch_time;
+  times[1].tv_usec = 0;
+
+  return lutimes(path.c_str(), times) == 0;
+}
+
+Path getCachePath(std::string key)
+{
+    return getCacheDir() + "/nix/gitv3/" +
+        hashString(htSHA256, key).to_string(Base32, false);
+}
+
+// Returns the name of the HEAD branch.
+//
+// Returns the head branch name as reported by git ls-remote --symref, e.g., if
+// ls-remote returns the output below, "main" is returned based on the ref line.
+//
+//   ref: refs/heads/main       HEAD
+//   ...
+std::optional<std::string> readHead(const Path & path)
+{
+    auto [exit_code, output] = runProgram(RunOptions {
+        .program = "git",
+        .args = {"ls-remote", "--symref", path},
+    });
+    if (exit_code != 0) {
+        return std::nullopt;
     }
-    return *gitDir;
+
+    std::string_view line = output;
+    line = line.substr(0, line.find("\n"));
+    if (const auto ref = parseListReferenceHeadRef(line); ref) {
+        debug("resolved HEAD ref '%s' for repo '%s'", *ref, path);
+        return *ref;
+    }
+    if (const auto rev = parseListReferenceForRev("HEAD", line); rev) {
+        debug("resolved HEAD rev '%s' for repo '%s'", *rev, path);
+        return *rev;
+    }
+    return std::nullopt;
 }
 
-static std::string readHead(const Path & path)
+// Persist the HEAD ref from the remote repo in the local cached repo.
+bool storeCachedHead(const std::string& actualUrl, const std::string& headRef)
 {
-    return chomp(runProgram("git", true, { "-C", path, "--git-dir", ".git", "rev-parse", "--abbrev-ref", "HEAD" }));
+    Path cacheDir = getCachePath(actualUrl);
+    try {
+        runProgram("git", true, { "-C", cacheDir, "symbolic-ref", "--", "HEAD", headRef });
+    } catch (ExecError &e) {
+        if (!WIFEXITED(e.status)) throw;
+        return false;
+    }
+    /* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */
+    return true;
 }
 
-static bool isNotDotGitDirectory(const Path & path)
+std::optional<std::string> readHeadCached(const std::string& actualUrl)
+{
+    // Create a cache path to store the branch of the HEAD ref. Append something
+    // in front of the URL to prevent collision with the repository itself.
+    Path cacheDir = getCachePath(actualUrl);
+    Path headRefFile = cacheDir + "/HEAD";
+
+    time_t now = time(0);
+    struct stat st;
+    std::optional<std::string> cachedRef;
+    if (stat(headRefFile.c_str(), &st) == 0) {
+        cachedRef = readHead(cacheDir);
+        if (cachedRef != std::nullopt &&
+            *cachedRef != gitInitialBranch &&
+            isCacheFileWithinTtl(now, st)) {
+            debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
+            return cachedRef;
+        }
+    }
+
+    auto ref = readHead(actualUrl);
+    if (ref) {
+        return ref;
+    }
+
+    if (cachedRef) {
+        // If the cached git ref is expired in fetch() below, and the 'git fetch'
+        // fails, it falls back to continuing with the most recent version.
+        // This function must behave the same way, so we return the expired
+        // cached ref here.
+        warn("could not get HEAD ref for repository '%s'; using expired cached ref '%s'", actualUrl, *cachedRef);
+        return *cachedRef;
+    }
+
+    return std::nullopt;
+}
+
+bool isNotDotGitDirectory(const Path & path)
 {
     return baseNameOf(path) != ".git";
 }
 
+struct WorkdirInfo
+{
+    bool clean = false;
+    bool hasHead = false;
+};
+
+// Returns whether a git workdir is clean and has commits.
+WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
+{
+    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+    auto gitDir = getGitDir();
+
+    auto env = getEnv();
+    // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
+    // that way unknown errors can lead to a failure instead of continuing through the wrong code path
+    env["LC_ALL"] = "C";
+
+    /* Check whether HEAD points to something that looks like a commit,
+       since that is the refrence we want to use later on. */
+    auto result = runProgram(RunOptions {
+        .program = "git",
+        .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
+        .environment = env,
+        .mergeStderrToStdout = true
+    });
+    auto exitCode = WEXITSTATUS(result.first);
+    auto errorMessage = result.second;
+
+    if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
+        throw Error("'%s' is not a Git repository", workdir);
+    } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
+        // indicates that the repo does not have any commits
+        // we want to proceed and will consider it dirty later
+    } else if (exitCode != 0) {
+        // any other errors should lead to a failure
+        throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage);
+    }
+
+    bool clean = false;
+    bool hasHead = exitCode == 0;
+
+    try {
+        if (hasHead) {
+            // Using git diff is preferrable over lower-level operations here,
+            // because its conceptually simpler and we only need the exit code anyways.
+            auto gitDiffOpts = Strings({ "-C", workdir, "diff", "HEAD", "--quiet"});
+            if (!submodules) {
+                // Changes in submodules should only make the tree dirty
+                // when those submodules will be copied as well.
+                gitDiffOpts.emplace_back("--ignore-submodules");
+            }
+            gitDiffOpts.emplace_back("--");
+            runProgram("git", true, gitDiffOpts);
+
+            clean = true;
+        }
+    } catch (ExecError & e) {
+        if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+    }
+
+    return WorkdirInfo { .clean = clean, .hasHead = hasHead };
+}
+
+std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
+{
+    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+
+    if (!fetchSettings.allowDirty)
+        throw Error("Git tree '%s' is dirty", workdir);
+
+    if (fetchSettings.warnDirty)
+        warn("Git tree '%s' is dirty", workdir);
+
+    auto gitOpts = Strings({ "-C", workdir, "ls-files", "-z" });
+    if (submodules)
+        gitOpts.emplace_back("--recurse-submodules");
+
+    auto files = tokenizeString<std::set<std::string>>(
+        runProgram("git", true, gitOpts), "\0"s);
+
+    Path actualPath(absPath(workdir));
+
+    PathFilter filter = [&](const Path & p) -> bool {
+        assert(hasPrefix(p, actualPath));
+        std::string file(p, actualPath.size() + 1);
+
+        auto st = lstat(p);
+
+        if (S_ISDIR(st.st_mode)) {
+            auto prefix = file + "/";
+            auto i = files.lower_bound(prefix);
+            return i != files.end() && hasPrefix(*i, prefix);
+        }
+
+        return files.count(file);
+    };
+
+    auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
+
+    // FIXME: maybe we should use the timestamp of the last
+    // modified dirty file?
+    input.attrs.insert_or_assign(
+        "lastModified",
+        workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
+
+    return {std::move(storePath), input};
+}
+}  // end namespace
+
 struct GitInputScheme : InputScheme
 {
     std::optional<Input> inputFromURL(const ParsedURL & url) override
@@ -234,106 +442,16 @@ struct GitInputScheme : InputScheme
         auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
-        // If this is a local directory and no ref or revision is
-        // given, then allow the use of an unclean working tree.
+        /* If this is a local directory and no ref or revision is given,
+           allow fetching directly from a dirty workdir. */
         if (!input.getRef() && !input.getRev() && isLocal) {
-            bool clean = false;
-
-            auto env = getEnv();
-            // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
-            // that way unknown errors can lead to a failure instead of continuing through the wrong code path
-            env["LC_ALL"] = "C";
-
-            /* Check whether HEAD points to something that looks like a commit,
-               since that is the refrence we want to use later on. */
-            auto result = runProgram(RunOptions {
-                .program = "git",
-                .args = { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
-                .environment = env,
-                .mergeStderrToStdout = true
-            });
-            auto exitCode = WEXITSTATUS(result.first);
-            auto errorMessage = result.second;
-
-            if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
-                throw Error("'%s' is not a Git repository", actualUrl);
-            } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
-                // indicates that the repo does not have any commits
-                // we want to proceed and will consider it dirty later
-            } else if (exitCode != 0) {
-                // any other errors should lead to a failure
-                throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage);
-            }
-
-            bool hasHead = exitCode == 0;
-            try {
-                if (hasHead) {
-                    // Using git diff is preferrable over lower-level operations here,
-                    // because its conceptually simpler and we only need the exit code anyways.
-                    auto gitDiffOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
-                    if (!submodules) {
-                        // Changes in submodules should only make the tree dirty
-                        // when those submodules will be copied as well.
-                        gitDiffOpts.emplace_back("--ignore-submodules");
-                    }
-                    gitDiffOpts.emplace_back("--");
-                    runProgram("git", true, gitDiffOpts);
-
-                    clean = true;
-                }
-            } catch (ExecError & e) {
-                if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-            }
-
-            if (!clean) {
-
-                /* This is an unclean working tree. So copy all tracked files. */
-
-                if (!fetchSettings.allowDirty)
-                    throw Error("Git tree '%s' is dirty", actualUrl);
-
-                if (fetchSettings.warnDirty)
-                    warn("Git tree '%s' is dirty", actualUrl);
-
-                auto gitOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "ls-files", "-z" });
-                if (submodules)
-                    gitOpts.emplace_back("--recurse-submodules");
-
-                auto files = tokenizeString<std::set<std::string>>(
-                    runProgram("git", true, gitOpts), "\0"s);
-
-                Path actualPath(absPath(actualUrl));
-
-                PathFilter filter = [&](const Path & p) -> bool {
-                    assert(hasPrefix(p, actualPath));
-                    std::string file(p, actualPath.size() + 1);
-
-                    auto st = lstat(p);
-
-                    if (S_ISDIR(st.st_mode)) {
-                        auto prefix = file + "/";
-                        auto i = files.lower_bound(prefix);
-                        return i != files.end() && hasPrefix(*i, prefix);
-                    }
-
-                    return files.count(file);
-                };
-
-                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
-
-                // FIXME: maybe we should use the timestamp of the last
-                // modified dirty file?
-                input.attrs.insert_or_assign(
-                    "lastModified",
-                    hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
-
-                return {std::move(storePath), input};
+            auto workdirInfo = getWorkdirInfo(input, actualUrl);
+            if (!workdirInfo.clean) {
+                return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
             }
         }
 
-        if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
-
-        Attrs unlockedAttrs({
+        const Attrs unlockedAttrs({
             {"type", cacheType},
             {"name", name},
             {"url", actualUrl},
@@ -343,14 +461,30 @@ struct GitInputScheme : InputScheme
         Path repoDir;
 
         if (isLocal) {
+            if (!input.getRef()) {
+                auto head = readHead(actualUrl);
+                if (!head) {
+                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
+                    head = "master";
+                }
+                input.attrs.insert_or_assign("ref", *head);
+            }
 
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev",
                     Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev());
 
             repoDir = actualUrl;
-
         } else {
+            const bool useHeadRef = !input.getRef();
+            if (useHeadRef) {
+                auto head = readHeadCached(actualUrl);
+                if (!head) {
+                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
+                    head = "master";
+                }
+                input.attrs.insert_or_assign("ref", *head);
+            }
 
             if (auto res = getCache()->lookup(store, unlockedAttrs)) {
                 auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
@@ -360,7 +494,7 @@ struct GitInputScheme : InputScheme
                 }
             }
 
-            Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
+            Path cacheDir = getCachePath(actualUrl);
             repoDir = cacheDir;
             gitDir = ".";
 
@@ -383,7 +517,7 @@ struct GitInputScheme : InputScheme
                repo. */
             if (input.getRev()) {
                 try {
-                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
+                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() });
                     doFetch = false;
                 } catch (ExecError & e) {
                     if (WIFEXITED(e.status)) {
@@ -400,7 +534,7 @@ struct GitInputScheme : InputScheme
                        git fetch to update the local ref to the remote ref. */
                     struct stat st;
                     doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-                        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+                        !isCacheFileWithinTtl(now, st);
                 }
             }
 
@@ -418,19 +552,16 @@ struct GitInputScheme : InputScheme
                             : ref == "HEAD"
                                 ? *ref
                                 : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
+                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
                 }
 
-                struct timeval times[2];
-                times[0].tv_sec = now;
-                times[0].tv_usec = 0;
-                times[1].tv_sec = now;
-                times[1].tv_usec = 0;
-
-                utimes(localRefFile.c_str(), times);
+                if (!touchCacheFile(localRefFile, now))
+                    warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
+                if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
+                    warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl);
             }
 
             if (!input.getRev())
@@ -459,7 +590,7 @@ struct GitInputScheme : InputScheme
 
         auto result = runProgram(RunOptions {
             .program = "git",
-            .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
+            .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
             .mergeStderrToStdout = true
         });
         if (WEXITSTATUS(result.first) == 128
@@ -498,7 +629,7 @@ struct GitInputScheme : InputScheme
             auto source = sinkToSource([&](Sink & sink) {
                 runProgram2({
                     .program = "git",
-                    .args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
+                    .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
                     .standardOut = &sink
                 });
             });
@@ -508,7 +639,7 @@ struct GitInputScheme : InputScheme
 
         auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
 
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},
@@ -517,7 +648,7 @@ struct GitInputScheme : InputScheme
 
         if (!shallow)
             infoAttrs.insert_or_assign("revCount",
-                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
+                std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() })));
 
         if (!_input.getRev())
             getCache()->add(

src/libfetchers/github.cc

@@ -4,7 +4,7 @@
 #include "store-api.hh"
 #include "types.hh"
 #include "url-parts.hh"
-
+#include "git-utils.hh"
 #include "fetchers.hh"
 #include "fetch-settings.hh"
 
@@ -383,35 +383,29 @@ struct SourceHutInputScheme : GitArchiveInputScheme
             std::string line;
             getline(is, line);
 
-            auto ref_index = line.find("ref: ");
-            if (ref_index == std::string::npos) {
+            auto r = parseListReferenceHeadRef(line);
+            if (!r) {
                 throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref);
             }
-
-            ref_uri = line.substr(ref_index+5, line.length()-1);
-        } else
+            ref_uri = *r;
+        } else {
             ref_uri = fmt("refs/(heads|tags)/%s", ref);
+        }
 
         auto file = store->toRealPath(
             downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
         std::ifstream is(file);
 
         std::string line;
-        std::string id;
-        while(getline(is, line)) {
-            // Append $ to avoid partial name matches
-            std::regex pattern(fmt("%s$", ref_uri));
-
-            if (std::regex_search(line, pattern)) {
-                id = line.substr(0, line.find('\t'));
-                break;
-            }
+        std::optional<std::string> id;
+        while(!id && getline(is, line)) {
+            id = parseListReferenceForRev(ref_uri, line);
         }
 
-        if(id.empty())
+        if(!id)
             throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
 
-        auto rev = Hash::parseAny(id, htSHA1);
+        auto rev = Hash::parseAny(*id, htSHA1);
         debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
         return rev;
     }

src/libstore/build/derivation-goal.cc

@@ -984,21 +984,28 @@ void DerivationGoal::resolvedFinished()
             realWantedOutputs = resolvedDrv.outputNames();
 
         for (auto & wantedOutput : realWantedOutputs) {
-            assert(initialOutputs.count(wantedOutput) != 0);
-            assert(resolvedHashes.count(wantedOutput) != 0);
-            auto realisation = resolvedResult.builtOutputs.at(
-                DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
+            auto initialOutput = get(initialOutputs, wantedOutput);
+            auto resolvedHash = get(resolvedHashes, wantedOutput);
+            if ((!initialOutput) || (!resolvedHash))
+                throw Error(
+                    "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
+                    worker.store.printStorePath(drvPath), wantedOutput);
+            auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
+            if (!realisation)
+                throw Error(
+                    "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
+                    worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
             if (drv->type().isPure()) {
-                auto newRealisation = realisation;
-                newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
+                auto newRealisation = *realisation;
+                newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
                 newRealisation.signatures.clear();
                 if (!drv->type().isFixed())
-                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
+                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
                 signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
             }
-            outputPaths.insert(realisation.outPath);
-            builtOutputs.emplace(realisation.id, realisation);
+            outputPaths.insert(realisation->outPath);
+            builtOutputs.emplace(realisation->id, *realisation);
         }
 
         runPostBuildHook(
@@ -1294,7 +1301,11 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
     DrvOutputs validOutputs;
 
     for (auto & i : queryPartialDerivationOutputMap()) {
-        InitialOutput & info = initialOutputs.at(i.first);
+        auto initialOutput = get(initialOutputs, i.first);
+        if (!initialOutput)
+            // this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
+            continue;
+        auto & info = *initialOutput;
         info.wanted = wantOutput(i.first, wantedOutputs);
         if (info.wanted)
             wantedOutputsLeft.erase(i.first);
@@ -1309,7 +1320,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
                     : PathStatus::Corrupt,
             };
         }
-        auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
+        auto drvOutput = DrvOutput{info.outputHash, i.first};
         if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
             if (auto real = worker.store.queryRealisation(drvOutput)) {
                 info.known = {

src/libstore/build/local-derivation-goal.cc

@@ -14,6 +14,7 @@
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
+#include "json-utils.hh"
 
 #include <regex>
 #include <queue>
@@ -56,8 +57,6 @@
 #include <pwd.h>
 #include <grp.h>
 
-#include <nlohmann/json.hpp>
-
 namespace nix {
 
 void handleDiffHook(
@@ -482,7 +481,7 @@ void LocalDerivationGoal::startBuilder()
            temporary build directory.  The text files have the format used
            by `nix-store --register-validity'.  However, the deriver
            fields are left empty. */
-        auto s = get(drv->env, "exportReferencesGraph").value_or("");
+        auto s = getOr(drv->env, "exportReferencesGraph", "");
         Strings ss = tokenizeString<Strings>(s);
         if (ss.size() % 2 != 0)
             throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
@@ -989,7 +988,7 @@ void LocalDerivationGoal::initTmpDir() {
        there is no size constraint). */
     if (!parsedDrv->getStructuredAttrs()) {
 
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
+        StringSet passAsFile = tokenizeString<StringSet>(getOr(drv->env, "passAsFile", ""));
         for (auto & i : drv->env) {
             if (passAsFile.find(i.first) == passAsFile.end()) {
                 env[i.first] = i.second;
@@ -2128,12 +2127,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
     std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
     std::map<std::string, struct stat> outputStats;
     for (auto & [outputName, _] : drv->outputs) {
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
+        auto scratchOutput = get(scratchOutputs, outputName);
+        if (!scratchOutput)
+            throw BuildError(
+                "builder for '%s' has no scratch output for '%s'",
+                worker.store.printStorePath(drvPath), outputName);
+        auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput));
 
         outputsToSort.insert(outputName);
 
         /* Updated wanted info to remove the outputs we definitely don't need to register */
-        auto & initialInfo = initialOutputs.at(outputName);
+        auto initialOutput = get(initialOutputs, outputName);
+        if (!initialOutput)
+            throw BuildError(
+                "builder for '%s' has no initial output for '%s'",
+                worker.store.printStorePath(drvPath), outputName);
+        auto & initialInfo = *initialOutput;
 
         /* Don't register if already valid, and not checking */
         initialInfo.wanted = buildMode == bmCheck
@@ -2185,6 +2194,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
 
     auto sortedOutputNames = topoSort(outputsToSort,
         {[&](const std::string & name) {
+            auto orifu = get(outputReferencesIfUnregistered, name);
+            if (!orifu)
+                throw BuildError(
+                    "no output reference for '%s' in build of '%s'",
+                    name, worker.store.printStorePath(drvPath));
             return std::visit(overloaded {
                 /* Since we'll use the already installed versions of these, we
                    can treat them as leaves and ignore any references they
@@ -2199,7 +2213,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                                 referencedOutputs.insert(o);
                     return referencedOutputs;
                 },
-            }, outputReferencesIfUnregistered.at(name));
+            }, *orifu);
         }},
         {[&](const std::string & path, const std::string & parent) {
             // TODO with more -vvvv also show the temporary paths for manual inspection.
@@ -2213,9 +2227,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
     OutputPathMap finalOutputs;
 
     for (auto & outputName : sortedOutputNames) {
-        auto output = drv->outputs.at(outputName);
-        auto & scratchPath = scratchOutputs.at(outputName);
-        auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
+        auto output = get(drv->outputs, outputName);
+        auto scratchPath = get(scratchOutputs, outputName);
+        assert(output && scratchPath);
+        auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath));
 
         auto finish = [&](StorePath finalStorePath) {
             /* Store the final path */
@@ -2223,10 +2238,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             /* The rewrite rule will be used in downstream outputs that refer to
                use. This is why the topological sort is essential to do first
                before this for loop. */
-            if (scratchPath != finalStorePath)
-                outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
+            if (*scratchPath != finalStorePath)
+                outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() };
         };
 
+        auto orifu = get(outputReferencesIfUnregistered, outputName);
+        assert(orifu);
+
         std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
             [&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> {
                 finish(skippedFinalPath.path);
@@ -2235,7 +2253,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             [&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> {
                 return r.refs;
             },
-        }, outputReferencesIfUnregistered.at(outputName));
+        }, *orifu);
 
         if (!referencesOpt)
             continue;
@@ -2268,25 +2286,29 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             for (auto & r : references) {
                 auto name = r.name();
                 auto origHash = std::string { r.hashPart() };
-                if (r == scratchPath)
+                if (r == *scratchPath) {
                     res.first = true;
-                else if (outputRewrites.count(origHash) == 0)
-                    res.second.insert(r);
-                else {
-                    std::string newRef = outputRewrites.at(origHash);
+                } else if (auto outputRewrite = get(outputRewrites, origHash)) {
+                    std::string newRef = *outputRewrite;
                     newRef += '-';
                     newRef += name;
                     res.second.insert(StorePath { newRef });
+                } else {
+                    res.second.insert(r);
                 }
             }
             return res;
         };
 
         auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
-            auto & st = outputStats.at(outputName);
+            auto st = get(outputStats, outputName);
+            if (!st)
+                throw BuildError(
+                    "output path %1% without valid stats info",
+                    actualPath);
             if (outputHash.method == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
-                if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
+                if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
                     throw BuildError(
                         "output path '%1%' should be a non-executable regular file "
                         "since recursive hashing is not enabled (outputHashMode=flat)",
@@ -2294,7 +2316,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             }
             rewriteOutput();
             /* FIXME optimize and deduplicate with addToStore */
-            std::string oldHashPart { scratchPath.hashPart() };
+            std::string oldHashPart { scratchPath->hashPart() };
             HashModuloSink caSink { outputHash.hashType, oldHashPart };
             switch (outputHash.method) {
             case FileIngestionMethod::Recursive:
@@ -2313,7 +2335,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                     outputPathName(drv->name, outputName),
                     refs.second,
                     refs.first);
-            if (scratchPath != finalPath) {
+            if (*scratchPath != finalPath) {
                 // Also rewrite the output path
                 auto source = sinkToSource([&](Sink & nextSink) {
                     StringSink sink;
@@ -2354,9 +2376,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                 auto requiredFinalPath = output.path;
                 /* Preemptively add rewrite rule for final hash, as that is
                    what the NAR hash will use rather than normalized-self references */
-                if (scratchPath != requiredFinalPath)
+                if (*scratchPath != requiredFinalPath)
                     outputRewrites.insert_or_assign(
-                        std::string { scratchPath.hashPart() },
+                        std::string { scratchPath->hashPart() },
                         std::string { requiredFinalPath.hashPart() });
                 rewriteOutput();
                 auto narHashAndSize = hashPath(htSHA256, actualPath);
@@ -2409,7 +2431,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                 });
             },
 
-        }, output.raw());
+        }, output->raw());
 
         /* FIXME: set proper permissions in restorePath() so
             we don't have to do another traversal. */
@@ -2425,7 +2447,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
            derivations. */
         PathLocks dynamicOutputLock;
         dynamicOutputLock.setDeletion(true);
-        auto optFixedPath = output.path(worker.store, drv->name, outputName);
+        auto optFixedPath = output->path(worker.store, drv->name, outputName);
         if (!optFixedPath ||
             worker.store.printStorePath(*optFixedPath) != finalDestPath)
         {
@@ -2491,11 +2513,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
 
         /* For debugging, print out the referenced and unreferenced paths. */
         for (auto & i : inputPaths) {
-            auto j = references.find(i);
-            if (j == references.end())
-                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
-            else
+            if (references.count(i))
                 debug("referenced input: '%1%'", worker.store.printStorePath(i));
+            else
+                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
         }
 
         if (curRound == nrRounds) {
@@ -2612,9 +2633,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
     DrvOutputs builtOutputs;
 
     for (auto & [outputName, newInfo] : infos) {
+        auto oldinfo = get(initialOutputs, outputName);
+        assert(oldinfo);
         auto thisRealisation = Realisation {
             .id = DrvOutput {
-                initialOutputs.at(outputName).outputHash,
+                oldinfo->outputHash,
                 outputName
             },
             .outPath = newInfo.path
@@ -2710,9 +2733,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
                 for (auto & i : *value) {
                     if (worker.store.isStorePath(i))
                         spec.insert(worker.store.parseStorePath(i));
-                    else if (outputs.count(i))
-                        spec.insert(outputs.at(i).path);
-                    else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
+                    else if (auto output = get(outputs, i))
+                        spec.insert(output->path);
+                    else
+                        throw BuildError("derivation contains an illegal reference specifier '%s'", i);
                 }
 
                 auto used = recursive
@@ -2751,24 +2775,18 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
         };
 
         if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            auto outputChecks = structuredAttrs->find("outputChecks");
-            if (outputChecks != structuredAttrs->end()) {
-                auto output = outputChecks->find(outputName);
-
-                if (output != outputChecks->end()) {
+            if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
+                if (auto output = get(*outputChecks, outputName)) {
                     Checks checks;
 
-                    auto maxSize = output->find("maxSize");
-                    if (maxSize != output->end())
+                    if (auto maxSize = get(*output, "maxSize"))
                         checks.maxSize = maxSize->get<uint64_t>();
 
-                    auto maxClosureSize = output->find("maxClosureSize");
-                    if (maxClosureSize != output->end())
+                    if (auto maxClosureSize = get(*output, "maxClosureSize"))
                         checks.maxClosureSize = maxClosureSize->get<uint64_t>();
 
-                    auto get = [&](const std::string & name) -> std::optional<Strings> {
-                        auto i = output->find(name);
-                        if (i != output->end()) {
+                    auto get_ = [&](const std::string & name) -> std::optional<Strings> {
+                        if (auto i = get(*output, name)) {
                             Strings res;
                             for (auto j = i->begin(); j != i->end(); ++j) {
                                 if (!j->is_string())
@@ -2781,10 +2799,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
                         return {};
                     };
 
-                    checks.allowedReferences = get("allowedReferences");
-                    checks.allowedRequisites = get("allowedRequisites");
-                    checks.disallowedReferences = get("disallowedReferences");
-                    checks.disallowedRequisites = get("disallowedRequisites");
+                    checks.allowedReferences = get_("allowedReferences");
+                    checks.allowedRequisites = get_("allowedRequisites");
+                    checks.disallowedReferences = get_("disallowedReferences");
+                    checks.disallowedRequisites = get_("disallowedRequisites");
 
                     applyChecks(checks);
                 }

src/libstore/build/worker.cc

@@ -350,7 +350,7 @@ void Worker::waitForInput()
        become `available'.  Note that `available' (i.e., non-blocking)
        includes EOF. */
     std::vector<struct pollfd> pollStatus;
-    std::map <int, int> fdToPollStatus;
+    std::map<int, size_t> fdToPollStatus;
     for (auto & i : children) {
         for (auto & j : i.fds) {
             pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
@@ -380,7 +380,10 @@ void Worker::waitForInput()
         std::set<int> fds2(j->fds);
         std::vector<unsigned char> buffer(4096);
         for (auto & k : fds2) {
-            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
+            const auto fdPollStatusId = get(fdToPollStatus, k);
+            assert(fdPollStatusId);
+            assert(*fdPollStatusId < pollStatus.size());
+            if (pollStatus.at(*fdPollStatusId).revents) {
                 ssize_t rd = ::read(k, buffer.data(), buffer.size());
                 // FIXME: is there a cleaner way to handle pt close
                 // than EIO? Is this even standard?

src/libstore/builtins/fetchurl.cc

@@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     Path storePath = getAttr("out");
     auto mainUrl = getAttr("url");
-    bool unpack = get(drv.env, "unpack").value_or("") == "1";
+    bool unpack = getOr(drv.env, "unpack", "") == "1";
 
     /* Note: have to use a fresh fileTransfer here because we're in
        a forked process. */

src/libstore/derivations.cc

@@ -661,8 +661,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         if (res.kind == DrvHash::Kind::Deferred)
             kind = DrvHash::Kind::Deferred;
         for (auto & outputName : inputOutputs) {
-            const auto h = res.hashes.at(outputName);
-            inputs2[h.to_string(Base16, false)].insert(outputName);
+            const auto h = get(res.hashes, outputName);
+            if (!h)
+                throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
+            inputs2[h->to_string(Base16, false)].insert(outputName);
         }
     }
 
@@ -836,8 +838,11 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
     auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
     for (auto & [outputName, output] : drv.outputs) {
         if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
-            auto & h = hashModulo.hashes.at(outputName);
-            auto outPath = store.makeOutputPath(outputName, h, drv.name);
+            auto h = get(hashModulo.hashes, outputName);
+            if (!h)
+                throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)",
+                    drv.name, outputName);
+            auto outPath = store.makeOutputPath(outputName, *h, drv.name);
             drv.env[outputName] = store.printStorePath(outPath);
             output = DerivationOutput::InputAddressed {
                 .path = std::move(outPath),

src/libstore/derived-path.cc

@@ -4,6 +4,8 @@
 
 #include <nlohmann/json.hpp>
 
+#include <optional>
+
 namespace nix {
 
 nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
@@ -17,12 +19,12 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
     res["drvPath"] = store->printStorePath(drvPath);
     // Fallback for the input-addressed derivation case: We expect to always be
     // able to print the output paths, so let’s do it
-    auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
+    const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
     for (const auto& output : outputs) {
-        if (knownOutputs.at(output))
-            res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value());
-        else
-            res["outputs"][output] = nullptr;
+        auto knownOutput = get(knownOutputs, output);
+        res["outputs"][output] = (knownOutput && *knownOutput)
+          ? store->printStorePath(**knownOutput)
+          : nullptr;
     }
     return res;
 }
@@ -123,10 +125,15 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
                 for (auto& [outputName, outputPath] : p.outputs) {
                     if (settings.isExperimentalFeatureEnabled(
                                 Xp::CaDerivations)) {
+                        auto drvOutput = get(drvHashes, outputName);
+                        if (!drvOutput)
+                            throw Error(
+                                "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
+                                store.printStorePath(p.drvPath), outputName);
                         auto thisRealisation = store.queryRealisation(
-                            DrvOutput{drvHashes.at(outputName), outputName});
-                        assert(thisRealisation);  // We’ve built it, so we must h
-                                                  // ve the realisation
+                            DrvOutput{*drvOutput, outputName});
+                        assert(thisRealisation);  // We’ve built it, so we must
+                                                  // have the realisation
                         res.insert(*thisRealisation);
                     } else {
                         res.insert(outputPath);

src/libstore/filetransfer.cc

@@ -692,10 +692,10 @@ struct curlFileTransfer : public FileTransfer
 #if ENABLE_S3
                 auto [bucketName, key, params] = parseS3Uri(request.uri);
 
-                std::string profile = get(params, "profile").value_or("");
-                std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1);
-                std::string scheme = get(params, "scheme").value_or("");
-                std::string endpoint = get(params, "endpoint").value_or("");
+                std::string profile = getOr(params, "profile", "");
+                std::string region = getOr(params, "region", Aws::Region::US_EAST_1);
+                std::string scheme = getOr(params, "scheme", "");
+                std::string endpoint = getOr(params, "endpoint", "");
 
                 S3Helper s3Helper(profile, region, scheme, endpoint);
 

src/libstore/local-store.cc

@@ -718,7 +718,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
                     // somewhat expensive so we do lazily
                     hashesModulo = hashDerivationModulo(*this, drv, true);
                 }
-                StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName);
+                auto currentOutputHash = get(hashesModulo->hashes, i.first);
+                if (!currentOutputHash)
+                    throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
+                        printStorePath(drvPath), printStorePath(doia.path), i.first);
+                StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName);
                 if (doia.path != recomputed)
                     throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
                         printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));

src/libstore/misc.cc

@@ -278,11 +278,16 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
     std::set<Realisation> inputRealisations;
 
     for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
-        auto outputHashes =
+        const auto outputHashes =
             staticOutputHashes(store, store.readDerivation(inputDrv));
         for (const auto & outputName : outputNames) {
+            auto outputHash = get(outputHashes, outputName);
+            if (!outputHash)
+                throw Error(
+                    "output '%s' of derivation '%s' isn't realised", outputName,
+                    store.printStorePath(inputDrv));
             auto thisRealisation = store.queryRealisation(
-                DrvOutput{outputHashes.at(outputName), outputName});
+                DrvOutput{*outputHash, outputName});
             if (!thisRealisation)
                 throw Error(
                     "output '%s' of derivation '%s' isn't built", outputName,

src/libstore/path-with-outputs.cc

@@ -1,5 +1,8 @@
 #include "path-with-outputs.hh"
 #include "store-api.hh"
+#include "nlohmann/json.hpp"
+
+#include <regex>
 
 namespace nix {
 
@@ -68,4 +71,57 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
     return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
 }
 
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
+{
+    static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
+
+    std::smatch match;
+    if (!std::regex_match(s, match, regex))
+        return {s, DefaultOutputs()};
+
+    if (match[3].matched)
+        return {match[1], AllOutputs()};
+
+    return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
+}
+
+std::string printOutputsSpec(const OutputsSpec & outputsSpec)
+{
+    if (std::get_if<DefaultOutputs>(&outputsSpec))
+        return "";
+
+    if (std::get_if<AllOutputs>(&outputsSpec))
+        return "^*";
+
+    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        return "^" + concatStringsSep(",", *outputNames);
+
+    assert(false);
+}
+
+void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
+{
+    if (std::get_if<DefaultOutputs>(&outputsSpec))
+        json = nullptr;
+
+    else if (std::get_if<AllOutputs>(&outputsSpec))
+        json = std::vector<std::string>({"*"});
+
+    else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        json = *outputNames;
+}
+
+void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
+{
+    if (json.is_null())
+        outputsSpec = DefaultOutputs();
+    else {
+        auto names = json.get<OutputNames>();
+        if (names == OutputNames({"*"}))
+            outputsSpec = AllOutputs();
+        else
+            outputsSpec = names;
+    }
+}
+
 }

src/libstore/path-with-outputs.hh

@@ -4,6 +4,7 @@
 
 #include "path.hh"
 #include "derived-path.hh"
+#include "nlohmann/json_fwd.hpp"
 
 namespace nix {
 
@@ -32,4 +33,25 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
 
 StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
 
+typedef std::set<std::string> OutputNames;
+
+struct AllOutputs {
+    bool operator < (const AllOutputs & _) const { return false; }
+};
+
+struct DefaultOutputs {
+    bool operator < (const DefaultOutputs & _) const { return false; }
+};
+
+typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
+
+/* Parse a string of the form 'prefix^output1,...outputN' or
+   'prefix^*', returning the prefix and the outputs spec. */
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
+
+std::string printOutputsSpec(const OutputsSpec & outputsSpec);
+
+void to_json(nlohmann::json &, const OutputsSpec &);
+void from_json(const nlohmann::json &, OutputsSpec &);
+
 }

src/libstore/remote-store.cc

@@ -853,15 +853,15 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
 
                         OutputPathMap outputs;
                         auto drv = evalStore->readDerivation(bfd.drvPath);
-                        auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
-                        auto drvOutputs = drv.outputsAndOptPaths(*this);
+                        const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
+                        const auto drvOutputs = drv.outputsAndOptPaths(*this);
                         for (auto & output : bfd.outputs) {
-                            if (!outputHashes.count(output))
+                            auto outputHash = get(outputHashes, output);
+                            if (!outputHash)
                                 throw Error(
                                     "the derivation '%s' doesn't have an output named '%s'",
                                     printStorePath(bfd.drvPath), output);
-                            auto outputId =
-                                DrvOutput{outputHashes.at(output), output};
+                            auto outputId = DrvOutput{ *outputHash, output };
                             if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
                                 auto realisation =
                                     queryRealisation(outputId);
@@ -874,13 +874,14 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
                             } else {
                                 // If ca-derivations isn't enabled, assume that
                                 // the output path is statically known.
-                                assert(drvOutputs.count(output));
-                                assert(drvOutputs.at(output).second);
+                                const auto drvOutput = get(drvOutputs, output);
+                                assert(drvOutput);
+                                assert(drvOutput->second);
                                 res.builtOutputs.emplace(
                                     outputId,
                                     Realisation {
                                         .id = outputId,
-                                        .outPath = *drvOutputs.at(output).second
+                                        .outPath = *drvOutput->second,
                                     });
                             }
                         }

src/libstore/store-api.cc

@@ -1314,7 +1314,7 @@ static bool isNonUriPath(const std::string & spec) {
 std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
 {
     if (uri == "" || uri == "auto") {
-        auto stateDir = get(params, "state").value_or(settings.nixStateDir);
+        auto stateDir = getOr(params, "state", settings.nixStateDir);
         if (access(stateDir.c_str(), R_OK | W_OK) == 0)
             return std::make_shared<LocalStore>(params);
         else if (pathExists(settings.nixDaemonSocketFile))

src/libstore/tests/path-with-outputs.cc

@@ -0,0 +1,46 @@
+#include "path-with-outputs.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+TEST(parseOutputsSpec, basic)
+{
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
+        ASSERT_EQ(prefix, "foo^bar");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
+        ASSERT_EQ(prefix, "foo^&*()");
+        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+    }
+}
+
+}

src/libutil/experimental-features.cc

@@ -35,7 +35,9 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
 
 std::string_view showExperimentalFeature(const ExperimentalFeature feature)
 {
-    return stringifiedXpFeatures.at(feature);
+    const auto ret = get(stringifiedXpFeatures, feature);
+    assert(ret);
+    return *ret;
 }
 
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
@@ -58,11 +60,13 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu
     return str << showExperimentalFeature(feature);
 }
 
-void to_json(nlohmann::json& j, const ExperimentalFeature& feature) {
+void to_json(nlohmann::json & j, const ExperimentalFeature & feature)
+{
     j = showExperimentalFeature(feature);
 }
 
-void from_json(const nlohmann::json& j, ExperimentalFeature& feature) {
+void from_json(const nlohmann::json & j, ExperimentalFeature & feature)
+{
     const std::string input = j;
     const auto parsed = parseExperimentalFeature(input);
 

src/libutil/experimental-features.hh

@@ -55,7 +55,7 @@ public:
  * Semi-magic conversion to and from json.
  * See the nlohmann/json readme for more details.
  */
-void to_json(nlohmann::json&, const ExperimentalFeature&);
-void from_json(const nlohmann::json&, ExperimentalFeature&);
+void to_json(nlohmann::json &, const ExperimentalFeature &);
+void from_json(const nlohmann::json &, ExperimentalFeature &);
 
 }

src/libutil/json-utils.hh

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+const nlohmann::json * get(const nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+nlohmann::json * get(nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+}

src/libutil/tests/tests.cc

@@ -548,7 +548,7 @@ namespace nix {
 
     TEST(get, emptyContainer) {
         StringMap s = { };
-        auto expected = std::nullopt;
+        auto expected = nullptr;
 
         ASSERT_EQ(get(s, "one"), expected);
     }
@@ -559,7 +559,23 @@ namespace nix {
         s["two"] = "er";
         auto expected = "yi";
 
-        ASSERT_EQ(get(s, "one"), expected);
+        ASSERT_EQ(*get(s, "one"), expected);
+    }
+
+    TEST(getOr, emptyContainer) {
+        StringMap s = { };
+        auto expected = "yi";
+
+        ASSERT_EQ(getOr(s, "one", "yi"), expected);
+    }
+
+    TEST(getOr, getFromContainer) {
+        StringMap s;
+        s["one"] = "yi";
+        s["two"] = "er";
+        auto expected = "yi";
+
+        ASSERT_EQ(getOr(s, "one", "nope"), expected);
     }
 
     /* ----------------------------------------------------------------------------

src/libutil/util.cc

@@ -1588,7 +1588,6 @@ std::string stripIndentation(std::string_view s)
 
 //////////////////////////////////////////////////////////////////////
 
-
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
 

src/libutil/util.hh

@@ -543,13 +543,31 @@ std::string stripIndentation(std::string_view s);
 
 /* Get a value for the specified key from an associate container. */
 template <class T>
-std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
+const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
 {
     auto i = map.find(key);
-    if (i == map.end()) return {};
-    return std::optional<typename T::mapped_type>(i->second);
+    if (i == map.end()) return nullptr;
+    return &i->second;
 }
 
+template <class T>
+typename T::mapped_type * get(T & map, const typename T::key_type & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &i->second;
+}
+
+/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */
+template <class T>
+const typename T::mapped_type & getOr(T & map,
+    const typename T::key_type & key,
+    const typename T::mapped_type & defaultValue)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return defaultValue;
+    return i->second;
+}
 
 /* Remove and return the first item from a container. */
 template <class T>

src/nix-build/nix-build.cc

@@ -440,7 +440,7 @@ static void main_nix_build(int argc, char * * argv)
         env["NIX_STORE"] = store->storeDir;
         env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
 
-        auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile").value_or(""));
+        auto passAsFile = tokenizeString<StringSet>(getOr(drv.env, "passAsFile", ""));
 
         bool keepTmp = false;
         int fileNr = 0;

src/nix/app.cc

@@ -89,7 +89,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
         auto outputName = cursor->getAttr(state.sOutputName)->getString();
         auto name = cursor->getAttr(state.sName)->getString();
         auto aPname = cursor->maybeGetAttr("pname");
-        auto aMeta = cursor->maybeGetAttr("meta");
+        auto aMeta = cursor->maybeGetAttr(state.sMeta);
         auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
         auto mainProgram =
             aMainProgram

src/nix/bundle.cc

@@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
 
         auto val = installable->toValue(*evalState).first;
 
-        auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
+        auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
         InstallableFlake bundler{this,
-            evalState, std::move(bundlerFlakeRef), bundlerName,
+            evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
             {"bundlers." + settings.thisSystem.get() + ".default",
              "defaultBundler." + settings.thisSystem.get()
             },

src/nix/develop.cc

@@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment
                 state,
                 installable->nixpkgsFlakeRef(),
                 "bashInteractive",
+                DefaultOutputs(),
                 Strings{},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 nixpkgsLockFlags);

src/nix/flake.cc

@@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
         auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
 
         auto installable = InstallableFlake(nullptr,
-            evalState, std::move(templateFlakeRef), templateName,
+            evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
             defaultTemplateAttrPaths,
             defaultTemplateAttrPathsPrefixes,
             lockFlags);
@@ -1015,8 +1015,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
                     auto name = visitor.getAttr(state->sName)->getString();
                     if (json) {
                         std::optional<std::string> description;
-                        if (auto aMeta = visitor.maybeGetAttr("meta")) {
-                            if (auto aDescription = aMeta->maybeGetAttr("description"))
+                        if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) {
+                            if (auto aDescription = aMeta->maybeGetAttr(state->sDescription))
                                 description = aDescription->getString();
                         }
                         j.emplace("type", "derivation");

src/nix/flake.md

@@ -153,7 +153,7 @@ Currently the `type` attribute can be one of the following:
   git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)?
   ```
 
-  The `ref` attribute defaults to `master`.
+  The `ref` attribute defaults to resolving the `HEAD` reference.
 
   The `rev` attribute must denote a commit that exists in the branch
   or tag specified by the `ref` attribute, since Nix doesn't do a full
@@ -161,6 +161,11 @@ Currently the `type` attribute can be one of the following:
   doesn't allow fetching a `rev` without a known `ref`). The default
   is the commit currently pointed to by `ref`.
 
+  When `git+file` is used without specifying `ref` or `rev`, files are
+  fetched directly from the local `path` as long as they have been added
+  to the Git repository. If there are uncommitted changes, the reference
+  is treated as dirty and a warning is printed.
+
   For example, the following are valid Git flake references:
 
   * `git+https://example.org/my/repo`

src/nix/nix.md

@@ -146,6 +146,51 @@ For most commands, if no installable is specified, the default is `.`,
 i.e. Nix will operate on the default flake output attribute of the
 flake in the current directory.
 
+## Derivation output selection
+
+Derivations can have multiple outputs, each corresponding to a
+different store path. For instance, a package can have a `bin` output
+that contains programs, and a `dev` output that provides development
+artifacts like C/C++ header files. The outputs on which `nix` commands
+operate are determined as follows:
+
+* You can explicitly specify the desired outputs using the syntax
+  *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can
+  obtain the `dev` and `static` outputs of the `glibc` package:
+
+  ```console
+  # nix build 'nixpkgs#glibc^dev,static'
+  # ls ./result-dev/include/ ./result-static/lib/
+  …
+  ```
+
+* You can also specify that *all* outputs should be used using the
+  syntax *installable*`^*`. For example, the following shows the size
+  of all outputs of the `glibc` package in the binary cache:
+
+  ```console
+  # nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
+  /nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123                 33208200
+  /nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin             36142896
+  /nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug          155787312
+  /nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static          42488328
+  /nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev             44200560
+  ```
+
+* If you didn't specify the desired outputs, but the derivation has an
+  attribute `meta.outputsToInstall`, Nix will use those outputs. For
+  example, since the package `nixpkgs#libxml2` has this attribute:
+
+  ```console
+  # nix eval 'nixpkgs#libxml2.meta.outputsToInstall'
+  [ "bin" "man" ]
+  ```
+
+  a command like `nix shell nixpkgs#libxml2` will provide only those
+  two outputs by default.
+
+* Otherwise, Nix will use all outputs of the derivation.
+
 # Nix stores
 
 Most `nix` subcommands operate on a *Nix store*.

src/nix/profile-install.md

@@ -20,6 +20,13 @@ R""(
   # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello
   ```
 
+* Install a specific output of a package:
+
+  ```console
+  # nix profile install nixpkgs#bash^man
+  ```
+
+
 # Description
 
 This command adds *installables* to a Nix profile.

src/nix/profile.cc

@@ -22,13 +22,13 @@ struct ProfileElementSource
     // FIXME: record original attrpath.
     FlakeRef resolvedRef;
     std::string attrPath;
-    // FIXME: output names
+    OutputsSpec outputs;
 
     bool operator < (const ProfileElementSource & other) const
     {
         return
-            std::pair(originalRef.to_string(), attrPath) <
-            std::pair(other.originalRef.to_string(), other.attrPath);
+            std::tuple(originalRef.to_string(), attrPath, outputs) <
+            std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs);
     }
 };
 
@@ -42,7 +42,7 @@ struct ProfileElement
     std::string describe() const
     {
         if (source)
-            return fmt("%s#%s", source->originalRef, source->attrPath);
+            return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs));
         StringSet names;
         for (auto & path : storePaths)
             names.insert(DrvName(path.name()).name);
@@ -67,7 +67,6 @@ struct ProfileElement
         ref<Store> store,
         const BuiltPaths & builtPaths)
     {
-        // FIXME: respect meta.outputsToInstall
         storePaths.clear();
         for (auto & buildable : builtPaths) {
             std::visit(overloaded {
@@ -99,7 +98,7 @@ struct ProfileManifest
             auto version = json.value("version", 0);
             std::string sUrl;
             std::string sOriginalUrl;
-            switch(version){
+            switch (version) {
                 case 1:
                     sUrl = "uri";
                     sOriginalUrl = "originalUri";
@@ -117,11 +116,12 @@ struct ProfileManifest
                 for (auto & p : e["storePaths"])
                     element.storePaths.insert(state.store->parseStorePath((std::string) p));
                 element.active = e["active"];
-                if (e.value(sUrl,"") != "") {
-                    element.source = ProfileElementSource{
+                if (e.value(sUrl, "") != "") {
+                    element.source = ProfileElementSource {
                         parseFlakeRef(e[sOriginalUrl]),
                         parseFlakeRef(e[sUrl]),
-                        e["attrPath"]
+                        e["attrPath"],
+                        e["outputs"].get<OutputsSpec>()
                     };
                 }
                 elements.emplace_back(std::move(element));
@@ -157,6 +157,7 @@ struct ProfileManifest
                 obj["originalUrl"] = element.source->originalRef.to_string();
                 obj["url"] = element.source->resolvedRef.to_string();
                 obj["attrPath"] = element.source->attrPath;
+                obj["outputs"] = element.source->outputs;
             }
             array.push_back(obj);
         }
@@ -284,10 +285,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
             if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
                 // FIXME: make build() return this?
                 auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
-                element.source = ProfileElementSource{
+                element.source = ProfileElementSource {
                     installable2->flakeRef,
                     resolvedRef,
                     attrPath,
+                    installable2->outputsSpec
                 };
             }
 
@@ -444,6 +446,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                     getEvalState(),
                     FlakeRef(element.source->originalRef),
                     "",
+                    element.source->outputs,
                     Strings{element.source->attrPath},
                     Strings{},
                     lockFlags);
@@ -455,10 +458,11 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 printInfo("upgrading '%s' from flake '%s' to '%s'",
                     element.source->attrPath, element.source->resolvedRef, resolvedRef);
 
-                element.source = ProfileElementSource{
+                element.source = ProfileElementSource {
                     installable->flakeRef,
                     resolvedRef,
                     attrPath,
+                    installable->outputsSpec
                 };
 
                 installables.push_back(installable);
@@ -514,8 +518,8 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
             logger->cout("%d %s %s %s", i,
-                element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-",
-                element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-",
+                element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
+                element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-",
                 concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
         }
     }

src/nix/search.cc

@@ -93,10 +93,10 @@ struct CmdSearch : InstallableCommand, MixJSON
                 };
 
                 if (cursor.isDerivation()) {
-                    DrvName name(cursor.getAttr("name")->getString());
+                    DrvName name(cursor.getAttr(state->sName)->getString());
 
-                    auto aMeta = cursor.maybeGetAttr("meta");
-                    auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
+                    auto aMeta = cursor.maybeGetAttr(state->sMeta);
+                    auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr;
                     auto description = aDescription ? aDescription->getString() : "";
                     std::replace(description.begin(), description.end(), '\n', ' ');
                     auto attrPath2 = concatStringsSep(".", attrPathS);

src/resolve-system-dependencies/resolve-system-dependencies.cc

@@ -176,7 +176,7 @@ int main(int argc, char ** argv)
             impurePaths.insert(argv[2]);
         else {
             auto drv = store->derivationFromPath(store->parseStorePath(argv[1]));
-            impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps").value_or(""));
+            impurePaths = tokenizeString<StringSet>(getOr(drv.env, "__impureHostDeps", ""));
             impurePaths.insert("/usr/lib/libSystem.dylib");
         }
 

tests/build-remote.sh

@@ -34,7 +34,7 @@ outPath=$(readlink -f $TEST_ROOT/result)
 
 grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath
 
-testPrintOutPath=$(nix build -L -v -f $file --print-out-paths --max-jobs 0 \
+testPrintOutPath=$(nix build -L -v -f $file --no-link --print-out-paths --max-jobs 0 \
   --arg busybox $busybox \
   --store $TEST_ROOT/machine0 \
   --builders "$(join_by '; ' "${builders[@]}")"
@@ -72,6 +72,7 @@ fi
 
 # Behavior of keep-failed
 out="$(nix-build 2>&1 failing.nix \
+  --no-out-link \
   --builders "$(join_by '; ' "${builders[@]}")"  \
   --keep-failed \
   --store $TEST_ROOT/machine0 \

tests/build.sh

@@ -2,15 +2,10 @@ source common.sh
 
 clearStore
 
-# Make sure that 'nix build' only returns the outputs we asked for.
-nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status '
-  (.[0] |
-    (.drvPath | match(".*multiple-outputs-a.drv")) and
-    (.outputs | keys | length == 1) and
-    (.outputs.first | match(".*multiple-outputs-a-first")))
-'
+set -o pipefail
 
-nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
+# Make sure that 'nix build' returns all outputs by default.
+nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
     (.outputs | keys | length == 2) and
@@ -22,6 +17,45 @@ nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-statu
     (.outputs.out | match(".*multiple-outputs-b")))
 '
 
+# Test output selection using the '^' syntax.
+nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys == ["first"]))
+'
+
+nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys == ["first", "second"]))
+'
+
+nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys == ["first", "second"]))
+'
+
+# Test that 'outputsToInstall' is respected by default.
+nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-e.drv")) and
+    (.outputs | keys == ["a", "b"]))
+'
+
+# But not when it's overriden.
+nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-e.drv")) and
+    (.outputs | keys == ["a"]))
+'
+
+nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-e.drv")) and
+    (.outputs | keys == ["a", "b", "c"]))
+'
+
 testNormalization () {
     clearStore
     outPath=$(nix-build ./simple.nix --no-out-link)

tests/ca/content-addressed.nix

@@ -23,7 +23,7 @@ rec {
   };
   rootCA = mkCADerivation {
     name = "rootCA";
-    outputs = [ "out" "dev" "foo"];
+    outputs = [ "out" "dev" "foo" ];
     buildCommand = ''
       echo "building a CA derivation"
       echo "The seed is ${toString seed}"

tests/ca/substitute.sh

@@ -25,7 +25,8 @@ buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transi
 # Check that the thing we’ve just substituted has its realisation stored
 nix realisation info --file ./content-addressed.nix transitivelyDependentCA
 # Check that its dependencies have it too
-nix realisation info --file ./content-addressed.nix dependentCA rootCA
+nix realisation info --file ./content-addressed.nix dependentCA
+# nix realisation info --file ./content-addressed.nix rootCA --outputs out
 
 # Same thing, but
 # 1. With non-ca derivations

tests/common.sh.in

@@ -157,11 +157,12 @@ expect() {
     local expected res
     expected="$1"
     shift
-    set +e
-    "$@"
-    res="$?"
-    set -e
-    [[ $res -eq $expected ]]
+    "$@" || res="$?"
+    if [[ $res -ne $expected ]]; then
+        echo "Expected '$expected' but got '$res' while running '$*'"
+        return 1
+    fi
+    return 0
 }
 
 needLocalStore() {

tests/fetchClosure.sh

@@ -7,7 +7,7 @@ clearStore
 clearCacheCache
 
 # Initialize binary cache.
-nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out)
+nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out)
 caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')
 nix copy --to file://$cacheDir $nonCaPath
 

tests/fetchGit.sh

@@ -161,6 +161,14 @@ path4=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
 [[ $(cat $path4/hello) = dev ]]
 [[ $path3 = $path4 ]]
 
+# Using remote path with branch other than 'master' should fetch the HEAD revision.
+# (--tarball-ttl 0 to prevent using the cached repo above)
+export _NIX_FORCE_HTTP=1
+path4=$(nix eval --tarball-ttl 0 --impure --raw --expr "(builtins.fetchGit $repo).outPath")
+[[ $(cat $path4/hello) = dev ]]
+[[ $path3 = $path4 ]]
+unset _NIX_FORCE_HTTP
+
 # Confirm same as 'dev' branch
 path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]

tests/flakes.sh

@@ -31,7 +31,14 @@ flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE
 for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB $flakeFollowsA; do
     rm -rf $repo $repo.tmp
     mkdir -p $repo
-    git -C $repo init
+
+    # Give one repo a non-master initial branch.
+    extraArgs=
+    if [[ $repo == $flake2Dir ]]; then
+      extraArgs="--initial-branch=main"
+    fi
+
+    git -C $repo init $extraArgs
     git -C $repo config user.email "foobar@example.com"
     git -C $repo config user.name "Foobar"
 done

tests/lang.sh

@@ -4,6 +4,7 @@ export TEST_VAR=foo # for eval-okay-getenv.nix
 export NIX_REMOTE=dummy://
 
 nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
+nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1
 (! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)
 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello
 
@@ -14,7 +15,7 @@ fail=0
 for i in lang/parse-fail-*.nix; do
     echo "parsing $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --parse - < lang/$i.nix; then
+    if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then
         echo "FAIL: $i shouldn't parse"
         fail=1
     fi
@@ -23,7 +24,7 @@ done
 for i in lang/parse-okay-*.nix; do
     echo "parsing $i (should succeed)";
     i=$(basename $i .nix)
-    if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
+    if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
         echo "FAIL: $i should parse"
         fail=1
     fi
@@ -32,7 +33,7 @@ done
 for i in lang/eval-fail-*.nix; do
     echo "evaluating $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --eval lang/$i.nix; then
+    if ! expect 1 nix-instantiate --eval lang/$i.nix; then
         echo "FAIL: $i shouldn't evaluate"
         fail=1
     fi
@@ -47,7 +48,7 @@ for i in lang/eval-okay-*.nix; do
         if test -e lang/$i.flags; then
             flags=$(cat lang/$i.flags)
         fi
-        if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
+        if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
             echo "FAIL: $i should evaluate"
             fail=1
         elif ! diff lang/$i.out lang/$i.exp; then
@@ -57,7 +58,7 @@ for i in lang/eval-okay-*.nix; do
     fi
 
     if test -e lang/$i.exp.xml; then
-        if ! nix-instantiate --eval --xml --no-location --strict \
+        if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
                 lang/$i.nix > lang/$i.out.xml; then
             echo "FAIL: $i should evaluate"
             fail=1

tests/multiple-outputs.nix

@@ -80,4 +80,11 @@ rec {
       '';
   }).a;
 
+  e = mkDerivation {
+    name = "multiple-outputs-e";
+    outputs = [ "a" "b" "c" ];
+    meta.outputsToInstall = [ "a" "b" ];
+    buildCommand = "mkdir $a $b $c";
+  };
+
 }

tests/nix-profile.sh

@@ -17,6 +17,7 @@ cat > $flake1Dir/flake.nix <<EOF
   outputs = { self }: with import ./config.nix; rec {
     packages.$system.default = mkDerivation {
       name = "profile-test-\${builtins.readFile ./version}";
+      outputs = [ "out" "man" "dev" ];
       builder = builtins.toFile "builder.sh"
         ''
           mkdir -p \$out/bin
@@ -26,10 +27,13 @@ cat > $flake1Dir/flake.nix <<EOF
           EOF
           chmod +x \$out/bin/hello
           echo DONE
+          mkdir -p \$man/share/man
+          mkdir -p \$dev/include
         '';
       __contentAddressed = import ./ca.nix;
       outputHashMode = "recursive";
       outputHashAlgo = "sha256";
+      meta.outputsToInstall = [ "out" "man" ];
     };
   };
 }
@@ -46,6 +50,8 @@ nix-env -f ./user-envs.nix -i foo-1.0
 nix profile list | grep '0 - - .*-foo-1.0'
 nix profile install $flake1Dir -L
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
+[ -e $TEST_HOME/.nix-profile/share/man ]
+(! [ -e $TEST_HOME/.nix-profile/include ])
 nix profile history
 nix profile history | grep "packages.$system.default: ∅ -> 1.0"
 nix profile diff-closures | grep 'env-manifest.nix: ε → ∅'
@@ -55,7 +61,7 @@ printf NixOS > $flake1Dir/who
 printf 2.0 > $flake1Dir/version
 nix profile upgrade 1
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]]
-nix profile history | grep "packages.$system.default: 1.0 -> 2.0"
+nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man"
 
 # Test 'history', 'diff-closures'.
 nix profile diff-closures
@@ -86,7 +92,7 @@ nix profile wipe-history
 printf true > $flake1Dir/ca.nix
 printf 3.0 > $flake1Dir/version
 nix profile upgrade 0
-nix profile history | grep "packages.$system.default: 1.0 -> 3.0"
+nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man"
 
 # Test new install of CA package.
 nix profile remove 0
@@ -95,3 +101,22 @@ printf Utrecht > $flake1Dir/who
 nix profile install $flake1Dir
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]]
 [[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]]
+
+# Override the outputs.
+nix profile remove 0 1
+nix profile install "$flake1Dir^*"
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]]
+[ -e $TEST_HOME/.nix-profile/share/man ]
+[ -e $TEST_HOME/.nix-profile/include ]
+
+printf Nix > $flake1Dir/who
+nix profile upgrade 0
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]]
+[ -e $TEST_HOME/.nix-profile/share/man ]
+[ -e $TEST_HOME/.nix-profile/include ]
+
+nix profile remove 0
+nix profile install "$flake1Dir^man"
+(! [ -e $TEST_HOME/.nix-profile/bin/hello ])
+[ -e $TEST_HOME/.nix-profile/share/man ]
+(! [ -e $TEST_HOME/.nix-profile/include ])

tests/shell-hello.nix

@@ -3,15 +3,24 @@ with import ./config.nix;
 {
   hello = mkDerivation {
     name = "hello";
+    outputs = [ "out" "dev" ];
+    meta.outputsToInstall = [ "out" ];
     buildCommand =
       ''
-        mkdir -p $out/bin
+        mkdir -p $out/bin $dev/bin
+
         cat > $out/bin/hello <<EOF
         #! ${shell}
         who=\$1
         echo "Hello \''${who:-World} from $out/bin/hello"
         EOF
         chmod +x $out/bin/hello
+
+        cat > $dev/bin/hello2 <<EOF
+        #! ${shell}
+        echo "Hello2"
+        EOF
+        chmod +x $dev/bin/hello2
       '';
   };
 }

tests/shell.sh

@@ -6,6 +6,10 @@ clearCache
 nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
 nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
 
+# Test output selection.
+nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2'
+nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2'
+
 if ! canUseSandbox; then exit 99; fi
 
 chmod -R u+w $TEST_ROOT/store0 || true