Merge remote-tracking branch 'nixos/master'

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v1.3.1
+        uses: zeebe-io/backport-action@v1.4.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -139,3 +139,6 @@ nix-rust/target
 result
 
 .vscode/
+
+# clangd and possibly more
+.cache/

doc/manual/src/advanced-topics/advanced-topics.md

@@ -1 +1 @@
-
+This section lists advanced topics related to builds and builds performance

doc/manual/src/glossary.md

@@ -1,6 +1,7 @@
 # Glossary
 
-  - [derivation]{#gloss-derivation}\
+- [derivation]{#gloss-derivation}
+
   A description of a build task. The result of a derivation is a
   store object. Derivations are typically specified in Nix expressions
   using the [`derivation` primitive](./language/derivations.md). These are
@@ -9,7 +10,8 @@
 
   [derivation]: #gloss-derivation
 
-  - [store derivation]{#gloss-store-derivation}\
+- [store derivation]{#gloss-store-derivation}
+
   A [derivation] represented as a `.drv` file in the [store].
   It has a [store path], like any [store object].
 
@@ -19,14 +21,16 @@
 
   [store derivation]: #gloss-store-derivation
 
-  - [instantiate]{#gloss-instantiate}, instantiation\
+- [instantiate]{#gloss-instantiate}, instantiation
+
   Translate a [derivation] into a [store derivation].
 
   See [`nix-instantiate`](./command-ref/nix-instantiate.md).
 
   [instantiate]: #gloss-instantiate
 
-  - [realise]{#gloss-realise}, realisation\
+- [realise]{#gloss-realise}, realisation
+
   Ensure a [store path] is [valid][validity].
 
   This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter].
@@ -37,16 +41,19 @@
 
   [realise]: #gloss-realise
 
-  - [content-addressed derivation]{#gloss-content-addressed-derivation}\
+- [content-addressed derivation]{#gloss-content-addressed-derivation}
+
   A derivation which has the
   [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed)
   attribute set to `true`.
 
-  - [fixed-output derivation]{#gloss-fixed-output-derivation}\
+- [fixed-output derivation]{#gloss-fixed-output-derivation}
+
   A derivation which includes the
   [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute.
 
-  - [store]{#gloss-store}\
+- [store]{#gloss-store}
+
   The location in the file system where store objects live. Typically
   `/nix/store`.
 
@@ -67,17 +74,20 @@
   [store]: #gloss-store
   [local store]: #gloss-local-store
 
-  - [chroot store]{#gloss-chroot-store}\
+- [chroot store]{#gloss-chroot-store}
+
   A [local store] whose canonical path is anything other than `/nix/store`.
 
-  - [binary cache]{#gloss-binary-cache}\
+- [binary cache]{#gloss-binary-cache}
+
   A *binary cache* is a Nix store which uses a different format: its
   metadata and signatures are kept in `.narinfo` files rather than in a
   [Nix database]. This different format simplifies serving store objects
   over the network, but cannot host builds. Examples of binary caches
   include S3 buckets and the [NixOS binary cache](https://cache.nixos.org).
 
-  - [store path]{#gloss-store-path}\
+- [store path]{#gloss-store-path}
+
   The location of a [store object] in the file system, i.e., an
   immediate child of the Nix store directory.
 
@@ -85,38 +95,44 @@
 
   [store path]: #gloss-store-path
 
-  - [file system object]{#gloss-store-object}\
+- [file system object]{#gloss-store-object}
+
   The Nix data model for representing simplified file system data.
 
   See [File System Object](@docroot@/architecture/file-system-object.md) for details.
 
   [file system object]: #gloss-file-system-object
 
-  - [store object]{#gloss-store-object}\
+- [store object]{#gloss-store-object}
+
 
   A store object consists of a [file system object], [reference]s to other store objects, and other metadata.
   It can be referred to by a [store path].
 
   [store object]: #gloss-store-object
 
-  - [input-addressed store object]{#gloss-input-addressed-store-object}\
+- [input-addressed store object]{#gloss-input-addressed-store-object}
+
   A store object produced by building a
   non-[content-addressed](#gloss-content-addressed-derivation),
   non-[fixed-output](#gloss-fixed-output-derivation)
   derivation.
 
-  - [output-addressed store object]{#gloss-output-addressed-store-object}\
+- [output-addressed store object]{#gloss-output-addressed-store-object}
+
   A [store object] whose [store path] is determined by its contents.
   This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
 
-  - [substitute]{#gloss-substitute}\
+- [substitute]{#gloss-substitute}
+
   A substitute is a command invocation stored in the [Nix database] that
   describes how to build a store object, bypassing the normal build
   mechanism (i.e., derivations). Typically, the substitute builds the
   store object by downloading a pre-built version of the store object
   from some server.
 
-  - [substituter]{#gloss-substituter}\
+- [substituter]{#gloss-substituter}
+
   An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them.
   Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter.
 
@@ -124,13 +140,15 @@
 
   [substituter]: #gloss-substituter
 
-  - [purity]{#gloss-purity}\
+- [purity]{#gloss-purity}
+
   The assumption that equal Nix derivations when run always produce
   the same output. This cannot be guaranteed in general (e.g., a
   builder can rely on external inputs such as the network or the
   system time) but the Nix model assumes it.
 
-  - [Nix database]{#gloss-nix-database}\
+- [Nix database]{#gloss-nix-database}
+
   An SQlite database to track [reference]s between [store object]s.
   This is an implementation detail of the [local store].
 
@@ -138,14 +156,16 @@
 
   [Nix database]: #gloss-nix-database
 
-  - [Nix expression]{#gloss-nix-expression}\
+- [Nix expression]{#gloss-nix-expression}
+
   A high-level description of software packages and compositions
   thereof. Deploying software using Nix entails writing Nix
   expressions for your packages. Nix expressions are translated to
   derivations that are stored in the Nix store. These derivations can
   then be built.
 
-  - [reference]{#gloss-reference}\
+- [reference]{#gloss-reference}
+
   A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`.
 
   Store objects can refer to both other store objects and themselves.
@@ -154,11 +174,13 @@
 
   [reference]: #gloss-reference
 
-  - [reachable]{#gloss-reachable}\
+- [reachable]{#gloss-reachable}
+
   A store path `Q` is reachable from another store path `P` if `Q`
   is in the *closure* of the *references* relation.
 
-  - [closure]{#gloss-closure}\
+- [closure]{#gloss-closure}
+
   The closure of a store path is the set of store paths that are
   directly or indirectly “reachable” from that store path; that is,
   it’s the closure of the path under the *references* relation. For
@@ -175,15 +197,18 @@
 
   [closure]: #gloss-closure
 
-  - [output path]{#gloss-output-path}\
+- [output path]{#gloss-output-path}
+
   A [store path] produced by a [derivation].
 
   [output path]: #gloss-output-path
 
-  - [deriver]{#gloss-deriver}\
+- [deriver]{#gloss-deriver}
+
   The [store derivation] that produced an [output path].
 
-  - [validity]{#gloss-validity}\
+- [validity]{#gloss-validity}
+
   A store path is valid if all [store object]s in its [closure] can be read from the [store].
 
   For a [local store], this means:
@@ -193,34 +218,41 @@
 
   [validity]: #gloss-validity
 
-  - [user environment]{#gloss-user-env}\
+- [user environment]{#gloss-user-env}
+
   An automatically generated store object that consists of a set of
   symlinks to “active” applications, i.e., other store paths. These
   are generated automatically by
   [`nix-env`](./command-ref/nix-env.md). See *profiles*.
 
-  - [profile]{#gloss-profile}\
+- [profile]{#gloss-profile}
+
   A symlink to the current *user environment* of a user, e.g.,
   `/nix/var/nix/profiles/default`.
 
-  - [installable]{#gloss-installable}\
+- [installable]{#gloss-installable}
+
   Something that can be realised in the Nix store.
 
   See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details.
 
-  - [NAR]{#gloss-nar}\
+- [NAR]{#gloss-nar}
+
   A *N*ix *AR*chive. This is a serialisation of a path in the Nix
   store. It can contain regular files, directories and symbolic
   links.  NARs are generated and unpacked using `nix-store --dump`
   and `nix-store --restore`.
 
-  - [`∅`]{#gloss-emtpy-set}\
+- [`∅`]{#gloss-emtpy-set}
+
   The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.
 
-  - [`ε`]{#gloss-epsilon}\
+- [`ε`]{#gloss-epsilon}
+
   The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.
 
-  - [string interpolation]{#gloss-string-interpolation}\
+- [string interpolation]{#gloss-string-interpolation}
+
   Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name].
 
   See [String interpolation](./language/string-interpolation.md) for details.
@@ -229,7 +261,8 @@
   [path]: ./language/values.md#type-path
   [attribute name]: ./language/values.md#attribute-set
 
-  - [experimental feature]{#gloss-experimental-feature}\
+- [experimental feature]{#gloss-experimental-feature}
+
   Not yet stabilized functionality guarded by named experimental feature flags.
   These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting.
 

doc/manual/src/release-notes/rl-next.md

@@ -19,3 +19,6 @@
 
 - The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field.
   This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op.
+
+- Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin.
+  It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

flake.nix

@@ -19,9 +19,11 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
+      linux32BitSystems = [ "i686-linux" ];
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
-      linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
+      linuxSystems = linux32BitSystems ++ linux64BitSystems;
-      systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ];
+      darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ];
+      systems = linuxSystems ++ darwinSystems;
       
       crossSystems = [ "armv6l-linux" "armv7l-linux" ];
 
@@ -40,6 +42,52 @@
             })
             stdenvs);
 
+      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
+      # Not an "idiomatic" flake input because:
+      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
+      #  - Subflake would download redundant and huge parent flake
+      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
+      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
+        fileset;
+
+      baseFiles =
+        # .gitignore has already been processed, so any changes in it are irrelevant
+        # at this point. It is not represented verbatim for test purposes because
+        # that would interfere with repo semantics.
+        fileset.fileFilter (f: f.name != ".gitignore") ./.;
+
+      nixSrc = fileset.toSource {
+        root = ./.;
+        fileset = fileset.intersect baseFiles (
+          fileset.difference
+            (fileset.unions [
+              ./.version
+              ./boehmgc-coroutine-sp-fallback.diff
+              ./bootstrap.sh
+              ./configure.ac
+              ./doc
+              ./local.mk
+              ./m4
+              ./Makefile
+              ./Makefile.config.in
+              ./misc
+              ./mk
+              ./precompiled-headers.h
+              ./src
+              ./tests
+              ./COPYING
+              ./scripts/local.mk
+              (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+              # TODO: do we really need README.md? It doesn't seem used in the build.
+              ./README.md
+            ])
+            (fileset.unions [
+              # Removed file sets
+              ./tests/nixos
+              ./tests/installer
+            ])
+        );
+      };
 
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
@@ -209,7 +257,7 @@
             "-${client.version}-against-${daemon.version}";
         inherit version;
 
-        src = self;
+        src = nixSrc;
 
         VERSION_SUFFIX = versionSuffix;
 
@@ -320,18 +368,11 @@
           };
           let
             canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
-
-            sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith {
-              filter = (path: type:
-                let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path);
-                in ! lib.any (re: builtins.match re relPath != null) rxs);
-              src = origSrc;
-            };
           in currentStdenv.mkDerivation (finalAttrs: {
             name = "nix-super-${version}";
             inherit version;
 
-            src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self;
+            src = nixSrc;
             VERSION_SUFFIX = versionSuffix;
 
             outputs = [ "out" "dev" "doc" ];
@@ -529,7 +570,7 @@
           releaseTools.coverageAnalysis {
             name = "nix-super-coverage-${version}";
 
-            src = self;
+            src = nixSrc;
 
             configureFlags = testConfigureFlags;
 
@@ -557,7 +598,7 @@
             pname = "nix-internal-api-docs";
             inherit version;
 
-            src = self;
+            src = nixSrc;
 
             configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags;
 

src/libcmd/installable-attr-path.cc

@@ -80,7 +80,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
             [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
                 return e;
             },
-        }, extendedOutputsSpec.raw());
+        }, extendedOutputsSpec.raw);
 
         auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs);
 
@@ -96,6 +96,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
                 .outputs = outputs,
             },
             .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {
+                .extendedOutputsSpec = outputs,
                 /* FIXME: reconsider backwards compatibility above
                    so we can fill in this info. */
             }),
@@ -114,7 +115,7 @@ InstallableAttrPath InstallableAttrPath::parse(
     return {
         state, cmd, v,
         prefix == "." ? "" : std::string { prefix },
-        extendedOutputsSpec
+        std::move(extendedOutputsSpec),
     };
 }
 

src/libcmd/installable-derived-path.cc

@@ -55,7 +55,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
                 .outputs = outputSpec,
             };
         },
-    }, extendedOutputsSpec.raw());
+    }, extendedOutputsSpec.raw);
     return InstallableDerivedPath {
         store,
         std::move(derivedPath),

src/libcmd/installable-flake.cc

@@ -141,7 +141,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
                 [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
                     return e;
                 },
-            }, extendedOutputsSpec.raw()),
+            }, extendedOutputsSpec.raw),
         },
         .info = make_ref<ExtraPathInfoFlake>(
             ExtraPathInfoValue::Value {

src/libcmd/installables.cc

@@ -583,7 +583,7 @@ Installables SourceExprCommand::parseInstallables(
         for (auto & s : ss) {
             auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s);
             auto installableAttr = make_ref<InstallableAttrPath>(InstallableAttrPath::parse(
-                state, *this, vFile, prefix, extendedOutputsSpec
+                state, *this, vFile, std::move(prefix), std::move(extendedOutputsSpec)
             ));
             if (doModifyInstallable) {
                 result.push_back(
@@ -611,7 +611,7 @@ Installables SourceExprCommand::parseInstallables(
             if (prefix.find('/') != std::string::npos) {
                 try {
                     result.push_back(make_ref<InstallableDerivedPath>(
-                        InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec)));
+                        InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec.raw)));
                     continue;
                 } catch (BadStorePath &) {
                 } catch (...) {
@@ -636,7 +636,7 @@ Installables SourceExprCommand::parseInstallables(
                         state,
                         std::move(flakeRef),
                         fragment,
-                        extendedOutputsSpec,
+                        std::move(extendedOutputsSpec),
                         getDefaultFlakeAttrPaths(),
                         getDefaultFlakeAttrPathPrefixes(),
                         lockFlags);

src/libcmd/modify-installables.cc

@@ -74,7 +74,7 @@ ref<Installable> SourceExprCommand::modifyInstallable (
     }
     return
         make_ref<InstallableAttrPath>(InstallableAttrPath::parse(
-            state, *this, vRes, prefix, extendedOutputsSpec
+            state, *this, vRes, std::move(prefix), std::move(extendedOutputsSpec)
         ));
 }
 

src/libexpr/eval-cache.cc

@@ -604,7 +604,7 @@ string_t AttrCursor::getStringWithContext()
                         [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
                             return o.path;
                         },
-                    }, c.raw());
+                    }, c.raw);
                     if (!root->state.store->isValidPath(path)) {
                         valid = false;
                         break;

src/libexpr/eval.cc

@@ -527,9 +527,9 @@ EvalState::EvalState(
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
         for (auto & i : _searchPath.elements)
-            addToSearchPath(SearchPath::Elem {i});
+            searchPath.elements.emplace_back(SearchPath::Elem {i});
         for (auto & i : evalSettings.nixPath.get())
-            addToSearchPath(SearchPath::Elem::parse(i));
+            searchPath.elements.emplace_back(SearchPath::Elem::parse(i));
     }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
@@ -1027,24 +1027,67 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
 }
 
 
+std::string EvalState::mkOutputStringRaw(
+    const SingleDerivedPath::Built & b,
+    std::optional<StorePath> optStaticOutputPath,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    /* In practice, this is testing for the case of CA derivations, or
+       dynamic derivations. */
+    return optStaticOutputPath
+        ? store->printStorePath(*std::move(optStaticOutputPath))
+        /* Downstream we would substitute this for an actual path once
+           we build the floating CA derivation */
+        : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
+}
+
+
 void EvalState::mkOutputString(
     Value & value,
-    const StorePath & drvPath,
+    const SingleDerivedPath::Built & b,
-    const std::string outputName,
+    std::optional<StorePath> optStaticOutputPath,
-    std::optional<StorePath> optOutputPath,
     const ExperimentalFeatureSettings & xpSettings)
 {
     value.mkString(
-        optOutputPath
+        mkOutputStringRaw(b, optStaticOutputPath, xpSettings),
-            ? store->printStorePath(*std::move(optOutputPath))
+        NixStringContext { b });
-            /* Downstream we would substitute this for an actual path once
+}
-               we build the floating CA derivation */
+
-            : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(),
+
-        NixStringContext {
+std::string EvalState::mkSingleDerivedPathStringRaw(
-            NixStringContextElem::Built {
+    const SingleDerivedPath & p)
-                .drvPath = makeConstantStorePathRef(drvPath),
+{
-                .output = outputName,
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & o) {
+            return store->printStorePath(o.path);
+        },
+        [&](const SingleDerivedPath::Built & b) {
+            auto optStaticOutputPath = std::visit(overloaded {
+                [&](const SingleDerivedPath::Opaque & o) {
+                    auto drv = store->readDerivation(o.path);
+                    auto i = drv.outputs.find(b.output);
+                    if (i == drv.outputs.end())
+                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
+                    return i->second.path(*store, drv.name, b.output);
+                },
+                [&](const SingleDerivedPath::Built & o) -> std::optional<StorePath> {
+                    return std::nullopt;
+                },
+            }, b.drvPath->raw());
+            return mkOutputStringRaw(b, optStaticOutputPath);
         }
+    }, p.raw());
+}
+
+
+void EvalState::mkSingleDerivedPathString(
+    const SingleDerivedPath & p,
+    Value & v)
+{
+    v.mkString(
+        mkSingleDerivedPathStringRaw(p),
+        NixStringContext {
+            std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
         });
 }
 
@@ -2321,7 +2364,7 @@ std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedP
         [&](NixStringContextElem::Built && b) -> SingleDerivedPath {
             return std::move(b);
         },
-    }, ((NixStringContextElem &&) *context.begin()).raw());
+    }, ((NixStringContextElem &&) *context.begin()).raw);
     return {
         std::move(derivedPath),
         std::move(s),
@@ -2333,39 +2376,25 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
 {
     auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
     auto s = s_;
+    auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
+    if (s != sExpected) {
+        /* `std::visit` is used here just to provide a more precise
+           error message. */
         std::visit(overloaded {
             [&](const SingleDerivedPath::Opaque & o) {
-            auto sExpected = store->printStorePath(o.path);
-            if (s != sExpected)
                 error(
                     "path string '%s' has context with the different path '%s'",
                     s, sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
             },
             [&](const SingleDerivedPath::Built & b) {
-            auto sExpected = std::visit(overloaded {
-                [&](const SingleDerivedPath::Opaque & o) {
-                    auto drv = store->readDerivation(o.path);
-                    auto i = drv.outputs.find(b.output);
-                    if (i == drv.outputs.end())
-                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
-                    auto optOutputPath = i->second.path(*store, drv.name, b.output);
-                    // This is testing for the case of CA derivations
-                    return optOutputPath
-                        ? store->printStorePath(*optOutputPath)
-                        : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
-                },
-                [&](const SingleDerivedPath::Built & o) {
-                    return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
-                },
-            }, b.drvPath->raw());
-            if (s != sExpected)
                 error(
                     "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
                     s, b.output, b.drvPath->to_string(*store), sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
             }
         }, derivedPath.raw());
+    }
     return derivedPath;
 }
 

src/libexpr/eval.hh

@@ -341,8 +341,6 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    void addToSearchPath(SearchPath::Elem && elem);
-
     SearchPath getSearchPath() { return searchPath; }
 
     /**
@@ -668,37 +666,46 @@ public:
     /**
      * Create a string representing a store path.
      *
-     * The string is the printed store path with a context containing a single
+     * The string is the printed store path with a context containing a
-     * `NixStringContextElem::Opaque` element of that store path.
+     * single `NixStringContextElem::Opaque` element of that store path.
      */
     void mkStorePathString(const StorePath & storePath, Value & v);
 
     /**
-     * Create a string representing a `DerivedPath::Built`.
+     * Create a string representing a `SingleDerivedPath::Built`.
      *
-     * The string is the printed store path with a context containing a single
+     * The string is the printed store path with a context containing a
-     * `NixStringContextElem::Built` element of the drv path and output name.
+     * single `NixStringContextElem::Built` element of the drv path and
+     * output name.
      *
      * @param value Value we are settings
      *
-     * @param drvPath Path the drv whose output we are making a string for
+     * @param b the drv whose output we are making a string for, and the
+     * output
      *
-     * @param outputName Name of the output
+     * @param optStaticOutputPath Optional output path for that string.
-     *
+     * Must be passed if and only if output store object is
-     * @param optOutputPath Optional output path for that string. Must
+     * input-addressed or fixed output. Will be printed to form string
-     * be passed if and only if output store object is input-addressed.
+     * if passed, otherwise a placeholder will be used (see
-     * Will be printed to form string if passed, otherwise a placeholder
+     * `DownstreamPlaceholder`).
-     * will be used (see `DownstreamPlaceholder`).
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     void mkOutputString(
         Value & value,
-        const StorePath & drvPath,
+        const SingleDerivedPath::Built & b,
-        const std::string outputName,
+        std::optional<StorePath> optStaticOutputPath,
-        std::optional<StorePath> optOutputPath,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
+    /**
+     * Create a string representing a `SingleDerivedPath`.
+     *
+     * A combination of `mkStorePathString` and `mkOutputString`.
+     */
+    void mkSingleDerivedPathString(
+        const SingleDerivedPath & p,
+        Value & v);
+
     void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**
@@ -714,6 +721,22 @@ public:
 
 private:
 
+    /**
+     * Like `mkOutputString` but just creates a raw string, not an
+     * string Value, which would also have a string context.
+     */
+    std::string mkOutputStringRaw(
+        const SingleDerivedPath::Built & b,
+        std::optional<StorePath> optStaticOutputPath,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+    /**
+     * Like `mkSingleDerivedPathStringRaw` but just creates a raw string
+     * Value, which would also have a string context.
+     */
+    std::string mkSingleDerivedPathStringRaw(
+        const SingleDerivedPath & p);
+
     unsigned long nrEnvs = 0;
     unsigned long nrValuesInEnvs = 0;
     unsigned long nrValues = 0;

src/libexpr/flake/flakeref.cc

@@ -246,7 +246,7 @@ std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragment
 {
     auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url);
     auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake);
-    return {std::move(flakeRef), fragment, extendedOutputsSpec};
+    return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)};
 }
 
 }

src/libexpr/parser.y

@@ -738,12 +738,6 @@ Expr * EvalState::parseStdin()
 }
 
 
-void EvalState::addToSearchPath(SearchPath::Elem && elem)
-{
-    searchPath.elements.emplace_back(std::move(elem));
-}
-
-
 SourcePath EvalState::findFile(const std::string_view path)
 {
     return findFile(searchPath, path);

src/libexpr/primops.cc

@@ -69,7 +69,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                 res.insert_or_assign(ctxS, ctxS);
                 ensureValid(d.drvPath);
             },
-        }, c.raw());
+        }, c.raw);
     }
 
     if (drvs.empty()) return {};
@@ -156,8 +156,10 @@ static void mkOutputString(
 {
     state.mkOutputString(
         attrs.alloc(o.first),
-        drvPath,
+        SingleDerivedPath::Built {
-        o.first,
+            .drvPath = makeConstantStorePathRef(drvPath),
+            .output = o.first,
+        },
         o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
 }
 
@@ -1263,7 +1265,7 @@ drvName, Bindings * attrs, Value & v)
             [&](const NixStringContextElem::Opaque & o) {
                 drv.inputSrcs.insert(o.path);
             },
-        }, c.raw());
+        }, c.raw);
     }
 
     /* Do we have all required attributes? */
@@ -1332,13 +1334,13 @@ drvName, Bindings * attrs, Value & v)
             if (isImpure)
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::Impure {
-                        .method = method.raw,
+                        .method = method,
                         .hashType = ht,
                     });
             else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
-                        .method = method.raw,
+                        .method = method,
                         .hashType = ht,
                     });
         }
@@ -1371,7 +1373,7 @@ drvName, Bindings * attrs, Value & v)
                 drv.env[i] = state.store->printStorePath(outPath);
                 drv.outputs.insert_or_assign(
                     i,
-                    DerivationOutputInputAddressed {
+                    DerivationOutput::InputAddressed {
                         .path = std::move(outPath),
                     });
             }
@@ -1379,7 +1381,7 @@ drvName, Bindings * attrs, Value & v)
             ;
         case DrvHash::Kind::Deferred:
             for (auto & i : outputs) {
-                drv.outputs.insert_or_assign(i, DerivationOutputDeferred {});
+                drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {});
             }
         }
     }
@@ -1836,6 +1838,45 @@ static RegisterPrimOp primop_readDir({
     .fun = prim_readDir,
 });
 
+/* Extend single element string context with another output. */
+static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
+
+    std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
+
+    state.mkSingleDerivedPathString(
+        SingleDerivedPath::Built {
+            .drvPath = make_ref<SingleDerivedPath>(drvPath),
+            .output = std::string { outputName },
+        },
+        v);
+}
+
+static RegisterPrimOp primop_outputOf({
+    .name = "__outputOf",
+    .args = {"derivation-reference", "output-name"},
+    .doc = R"(
+      Return the output path of a derivation, literally or using a placeholder if needed.
+
+      If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned.
+      But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead.
+
+      *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
+      This primop can be chained arbitrarily deeply.
+      For instance,
+      ```nix
+      builtins.outputOf
+        (builtins.outputOf myDrv "out)
+        "out"
+      ```
+      will return a placeholder for the output of the output of `myDrv`.
+
+      This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.
+    )",
+    .fun = prim_outputOf,
+    .experimentalFeature = Xp::DynamicDerivations,
+});
 
 /*************************************************************
  * Creating files
@@ -2013,7 +2054,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
     StorePathSet refs;
 
     for (auto c : context) {
-        if (auto p = std::get_if<NixStringContextElem::Opaque>(&c))
+        if (auto p = std::get_if<NixStringContextElem::Opaque>(&c.raw))
             refs.insert(p->path);
         else
             state.debugThrowLastTrace(EvalError({

src/libexpr/primops/context.cc

@@ -51,13 +51,13 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
 
     NixStringContext context2;
     for (auto && c : context) {
-        if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) {
+        if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c.raw)) {
             context2.emplace(NixStringContextElem::Opaque {
                 .path = ptr->drvPath
             });
         } else {
             /* Can reuse original item */
-            context2.emplace(std::move(c));
+            context2.emplace(std::move(c).raw);
         }
     }
 
@@ -114,7 +114,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
             [&](NixStringContextElem::Opaque && o) {
                 contextInfos[std::move(o.path)].path = true;
             },
-        }, ((NixStringContextElem &&) i).raw());
+        }, ((NixStringContextElem &&) i).raw);
     }
 
     auto attrs = state.buildBindings(contextInfos.size());

src/libexpr/search-path.cc

@@ -10,7 +10,7 @@ std::optional<std::string_view> SearchPath::Prefix::suffixIfPotentialMatch(
 
     /* Non-empty prefix and suffix must be separated by a /, or the
        prefix is not a valid path prefix. */
-    bool needSeparator = n > 0 && (path.size() - n) > 0;
+    bool needSeparator = n > 0 && n < path.size();
 
     if (needSeparator && path[n] != '/') {
         return std::nullopt;

src/libexpr/tests/derived-path.cc

@@ -34,8 +34,8 @@ RC_GTEST_FIXTURE_PROP(
 
 RC_GTEST_FIXTURE_PROP(
     DerivedPathExpressionTest,
-    prop_built_path_placeholder_round_trip,
+    prop_derived_path_built_placeholder_round_trip,
-    (const StorePath & drvPath, const StorePathName & outputName))
+    (const SingleDerivedPath::Built & b))
 {
     /**
      * We set these in tests rather than the regular globals so we don't have
@@ -45,27 +45,19 @@ RC_GTEST_FIXTURE_PROP(
     mockXpSettings.set("experimental-features", "ca-derivations");
 
     auto * v = state.allocValue();
-    state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings);
+    state.mkOutputString(*v, b, std::nullopt, mockXpSettings);
     auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
-    SingleDerivedPath::Built b {
-        .drvPath = makeConstantStorePathRef(drvPath),
-        .output = outputName.name,
-    };
     RC_ASSERT(SingleDerivedPath { b } == d);
 }
 
 RC_GTEST_FIXTURE_PROP(
     DerivedPathExpressionTest,
-    prop_built_path_out_path_round_trip,
+    prop_derived_path_built_out_path_round_trip,
-    (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath))
+    (const SingleDerivedPath::Built & b, const StorePath & outPath))
 {
     auto * v = state.allocValue();
-    state.mkOutputString(*v, drvPath, outputName.name, outPath);
+    state.mkOutputString(*v, b, outPath);
     auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
-    SingleDerivedPath::Built b {
-        .drvPath = makeConstantStorePathRef(drvPath),
-        .output = outputName.name,
-    };
     RC_ASSERT(SingleDerivedPath { b } == d);
 }
 

src/libexpr/tests/flakeref.cc

@@ -0,0 +1,22 @@
+#include <gtest/gtest.h>
+
+#include "flake/flakeref.hh"
+
+namespace nix {
+
+/* ----------- tests for flake/flakeref.hh --------------------------------------------------*/
+
+    /* ----------------------------------------------------------------------------
+     * to_string
+     * --------------------------------------------------------------------------*/
+
+    TEST(to_string, doesntReencodeUrl) {
+        auto s = "http://localhost:8181/test/+3d.tar.gz";
+        auto flakeref = parseFlakeRef(s);
+        auto parsed = flakeref.to_string();
+        auto expected = "http://localhost:8181/test/%2B3d.tar.gz";
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+}

src/libexpr/tests/value/context.cc

@@ -47,7 +47,7 @@ TEST(NixStringContextElemTest, slash_invalid) {
 TEST(NixStringContextElemTest, opaque) {
     std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
     auto elem = NixStringContextElem::parse(opaque);
-    auto * p = std::get_if<NixStringContextElem::Opaque>(&elem);
+    auto * p = std::get_if<NixStringContextElem::Opaque>(&elem.raw);
     ASSERT_TRUE(p);
     ASSERT_EQ(p->path, StorePath { opaque });
     ASSERT_EQ(elem.to_string(), opaque);
@@ -60,7 +60,7 @@ TEST(NixStringContextElemTest, opaque) {
 TEST(NixStringContextElemTest, drvDeep) {
     std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
     auto elem = NixStringContextElem::parse(drvDeep);
-    auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem);
+    auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem.raw);
     ASSERT_TRUE(p);
     ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) });
     ASSERT_EQ(elem.to_string(), drvDeep);
@@ -73,7 +73,7 @@ TEST(NixStringContextElemTest, drvDeep) {
 TEST(NixStringContextElemTest, built_opaque) {
     std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
     auto elem = NixStringContextElem::parse(built);
-    auto * p = std::get_if<NixStringContextElem::Built>(&elem);
+    auto * p = std::get_if<NixStringContextElem::Built>(&elem.raw);
     ASSERT_TRUE(p);
     ASSERT_EQ(p->output, "foo");
     ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
@@ -96,7 +96,7 @@ TEST(NixStringContextElemTest, built_built) {
 
     std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
     auto elem = NixStringContextElem::parse(built, mockXpSettings);
-    auto * p = std::get_if<NixStringContextElem::Built>(&elem);
+    auto * p = std::get_if<NixStringContextElem::Built>(&elem.raw);
     ASSERT_TRUE(p);
     ASSERT_EQ(p->output, "foo");
     auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath);

src/libexpr/value/context.cc

@@ -99,7 +99,7 @@ std::string NixStringContextElem::to_string() const
             res += '=';
             res += d.drvPath.to_string();
         },
-    }, raw());
+    }, raw);
 
     return res;
 }

src/libexpr/value/context.hh

@@ -4,8 +4,7 @@
 #include "util.hh"
 #include "comparator.hh"
 #include "derived-path.hh"
-
+#include "variant-wrapper.hh"
-#include <variant>
 
 #include <nlohmann/json_fwd.hpp>
 
@@ -26,14 +25,15 @@ public:
     }
 };
 
-/**
+struct NixStringContextElem {
+    /**
      * Plain opaque path to some store object.
      *
      * Encoded as just the path: ‘<path>’.
      */
-typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque;
+    using Opaque = SingleDerivedPath::Opaque;
 
-/**
+    /**
      * Path to a derivation and its entire build closure.
      *
      * The path doesn't just refer to derivation itself and its closure, but
@@ -42,42 +42,30 @@ typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque;
      *
      * Encoded in the form ‘=<drvPath>’.
      */
-struct NixStringContextElem_DrvDeep {
+    struct DrvDeep {
         StorePath drvPath;
 
-    GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath);
+        GENERATE_CMP(DrvDeep, me->drvPath);
-};
+    };
 
-/**
+    /**
      * Derivation output.
      *
      * Encoded in the form ‘!<output>!<drvPath>’.
      */
-typedef SingleDerivedPath::Built NixStringContextElem_Built;
+    using Built = SingleDerivedPath::Built;
 
-using _NixStringContextElem_Raw = std::variant<
+    using Raw = std::variant<
-    NixStringContextElem_Opaque,
+        Opaque,
-    NixStringContextElem_DrvDeep,
+        DrvDeep,
-    NixStringContextElem_Built
+        Built
->;
+    >;
 
-struct NixStringContextElem : _NixStringContextElem_Raw {
+    Raw raw;
-    using Raw = _NixStringContextElem_Raw;
-    using Raw::Raw;
 
-    using Opaque = NixStringContextElem_Opaque;
+    GENERATE_CMP(NixStringContextElem, me->raw);
-    using DrvDeep = NixStringContextElem_DrvDeep;
-    using Built = NixStringContextElem_Built;
 
-    inline const Raw & raw() const & {
+    MAKE_WRAPPER_CONSTRUCTOR(NixStringContextElem);
-        return static_cast<const Raw &>(*this);
-    }
-    inline Raw & raw() & {
-        return static_cast<Raw &>(*this);
-    }
-    inline Raw && raw() && {
-        return static_cast<Raw &&>(*this);
-    }
 
     /**
      * Decode a context string, one of:

src/libmain/progress-bar.cc

@@ -108,7 +108,8 @@ public:
         stop();
     }
 
-    void stop() override
+    /* Called by destructor, can't be overridden */
+    void stop() override final
     {
         {
             auto state(state_.lock());

src/libstore/build/derivation-goal.cc

@@ -521,7 +521,7 @@ void DerivationGoal::inputsRealised()
             [&](const DerivationType::Impure &) {
                 return true;
             }
-        }, drvType.raw());
+        }, drvType.raw);
 
         if (resolveDrv && !fullDrv.inputDrvs.empty()) {
             experimentalFeatureSettings.require(Xp::CaDerivations);
@@ -996,10 +996,11 @@ void DerivationGoal::buildDone()
         }
 
         else {
+            assert(derivationType);
             st =
                 dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
                 statusOk(status) ? BuildResult::OutputRejected :
-                !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
+                !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure :
                 BuildResult::PermanentFailure;
         }
 
@@ -1358,7 +1359,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
         [&](const OutputsSpec::Names & names) {
             return static_cast<StringSet>(names);
         },
-    }, wantedOutputs.raw());
+    }, wantedOutputs.raw);
     SingleDrvOutputs validOutputs;
 
     for (auto & i : queryPartialDerivationOutputMap()) {

src/libstore/build/derivation-goal.hh

@@ -184,7 +184,7 @@ struct DerivationGoal : public Goal
     /**
      * The sort of derivation we are building.
      */
-    DerivationType derivationType;
+    std::optional<DerivationType> derivationType;
 
     typedef void (DerivationGoal::*GoalState)();
     GoalState state;

src/libstore/build/goal.hh

@@ -162,6 +162,10 @@ public:
 
     virtual void cleanup() { }
 
+    /**
+     * @brief Hint for the scheduler, which concurrency limit applies.
+     * @see JobCategory
+     */
     virtual JobCategory jobCategory() = 0;
 };
 

src/libstore/build/local-derivation-goal.cc

@@ -178,6 +178,8 @@ void LocalDerivationGoal::tryLocalBuild()
         return;
     }
 
+    assert(derivationType);
+
     /* Are we doing a chroot build? */
     {
         auto noChroot = parsedDrv->getBoolAttr("__noChroot");
@@ -195,7 +197,7 @@ void LocalDerivationGoal::tryLocalBuild()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = derivationType.isSandboxed() && !noChroot;
+            useChroot = derivationType->isSandboxed() && !noChroot;
     }
 
     auto & localStore = getLocalStore();
@@ -689,7 +691,7 @@ void LocalDerivationGoal::startBuilder()
                 "nogroup:x:65534:\n", sandboxGid()));
 
         /* Create /etc/hosts with localhost entry. */
-        if (derivationType.isSandboxed())
+        if (derivationType->isSandboxed())
             writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
@@ -893,7 +895,7 @@ void LocalDerivationGoal::startBuilder()
            us.
         */
 
-        if (derivationType.isSandboxed())
+        if (derivationType->isSandboxed())
             privateNetwork = true;
 
         userNamespaceSync.create();
@@ -1121,7 +1123,7 @@ void LocalDerivationGoal::initEnv()
        derivation, tell the builder, so that for instance `fetchurl'
        can skip checking the output.  On older Nixes, this environment
        variable won't be set, so `fetchurl' will do the check. */
-    if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1";
+    if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1";
 
     /* *Only* if this is a fixed-output derivation, propagate the
        values of the environment variables specified in the
@@ -1132,7 +1134,7 @@ void LocalDerivationGoal::initEnv()
        to the builder is generally impure, but the output of
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
-    if (!derivationType.isSandboxed()) {
+    if (!derivationType->isSandboxed()) {
         for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
             env[i] = getEnv(i).value_or("");
     }
@@ -1797,7 +1799,7 @@ void LocalDerivationGoal::runChild()
             /* Fixed-output derivations typically need to access the
                network, so give them access to /etc/resolv.conf and so
                on. */
-            if (!derivationType.isSandboxed()) {
+            if (!derivationType->isSandboxed()) {
                 // Only use nss functions to resolve hosts and
                 // services. Don’t use it for anything else that may
                 // be configured for this system. This limits the
@@ -2048,7 +2050,7 @@ void LocalDerivationGoal::runChild()
                     #include "sandbox-defaults.sb"
                     ;
 
-                if (!derivationType.isSandboxed())
+                if (!derivationType->isSandboxed())
                     sandboxProfile +=
                         #include "sandbox-network.sb"
                         ;
@@ -2599,7 +2601,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 });
             },
 
-        }, output->raw());
+        }, output->raw);
 
         /* FIXME: set proper permissions in restorePath() so
             we don't have to do another traversal. */

src/libstore/build/local-derivation-goal.hh

@@ -272,8 +272,10 @@ struct LocalDerivationGoal : public DerivationGoal
 
     /**
      * Forcibly kill the child process, if any.
+     *
+     * Called by destructor, can't be overridden
      */
-    void killChild() override;
+    void killChild() override final;
 
     /**
      * Kill any processes running under the build user UID or in the

src/libstore/build/substitution-goal.hh

@@ -114,7 +114,8 @@ public:
     void handleChildOutput(int fd, std::string_view data) override;
     void handleEOF(int fd) override;
 
-    void cleanup() override;
+    /* Called by destructor, can't be overridden */
+    void cleanup() override final;
 
     JobCategory jobCategory() override { return JobCategory::Substitution; };
 };

src/libstore/build/worker.cc

@@ -268,7 +268,10 @@ void Worker::run(const Goals & _topGoals)
     for (auto & i : _topGoals) {
         topGoals.insert(i);
         if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
-            topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs});
+            topPaths.push_back(DerivedPath::Built {
+                .drvPath = makeConstantStorePathRef(goal->drvPath),
+                .outputs = goal->wantedOutputs,
+            });
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
             topPaths.push_back(DerivedPath::Opaque{goal->storePath});
         }

src/libstore/content-address.cc

@@ -115,7 +115,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
     auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest);
 
     return ContentAddress {
-        .method = std::move(caMethod).raw,
+        .method = std::move(caMethod),
         .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
     };
 }

src/libstore/content-address.hh

@@ -5,6 +5,7 @@
 #include "hash.hh"
 #include "path.hh"
 #include "comparator.hh"
+#include "variant-wrapper.hh"
 
 namespace nix {
 
@@ -71,11 +72,7 @@ struct ContentAddressMethod
 
     GENERATE_CMP(ContentAddressMethod, me->raw);
 
-    /* The moral equivalent of `using Raw::Raw;` */
+    MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod);
-    ContentAddressMethod(auto &&... arg)
-        : raw(std::forward<decltype(arg)>(arg)...)
-    { }
-
 
     /**
      * Parse the prefix tag which indicates how the files
@@ -252,10 +249,7 @@ struct ContentAddressWithReferences
 
     GENERATE_CMP(ContentAddressWithReferences, me->raw);
 
-    /* The moral equivalent of `using Raw::Raw;` */
+    MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences);
-    ContentAddressWithReferences(auto &&... arg)
-        : raw(std::forward<decltype(arg)>(arg)...)
-    { }
 
     /**
      * Create a `ContentAddressWithReferences` from a mere

src/libstore/derivations.cc

@@ -32,7 +32,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
         [](const DerivationOutput::Impure &) -> std::optional<StorePath> {
             return std::nullopt;
         },
-    }, raw());
+    }, raw);
 }
 
 
@@ -60,7 +60,7 @@ bool DerivationType::isCA() const
         [](const Impure &) {
             return true;
         },
-    }, raw());
+    }, raw);
 }
 
 bool DerivationType::isFixed() const
@@ -75,7 +75,7 @@ bool DerivationType::isFixed() const
         [](const Impure &) {
             return false;
         },
-    }, raw());
+    }, raw);
 }
 
 bool DerivationType::hasKnownOutputPaths() const
@@ -90,7 +90,7 @@ bool DerivationType::hasKnownOutputPaths() const
         [](const Impure &) {
             return false;
         },
-    }, raw());
+    }, raw);
 }
 
 
@@ -106,7 +106,7 @@ bool DerivationType::isSandboxed() const
         [](const Impure &) {
             return false;
         },
-    }, raw());
+    }, raw);
 }
 
 
@@ -122,7 +122,7 @@ bool DerivationType::isPure() const
         [](const Impure &) {
             return false;
         },
-    }, raw());
+    }, raw);
 }
 
 
@@ -408,13 +408,13 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
             },
-            [&](const DerivationOutputImpure & doi) {
+            [&](const DerivationOutput::Impure & doi) {
                 // FIXME
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
                 s += ','; printUnquotedString(s, "impure");
             }
-        }, i.second.raw());
+        }, i.second.raw);
         s += ')';
     }
 
@@ -509,7 +509,7 @@ DerivationType BasicDerivation::type() const
             [&](const DerivationOutput::Impure &) {
                 impureOutputs.insert(i.first);
             },
-        }, i.second.raw());
+        }, i.second.raw);
     }
 
     if (inputAddressedOutputs.empty()
@@ -626,7 +626,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
     if (type.isFixed()) {
         std::map<std::string, Hash> outputHashes;
         for (const auto & i : drv.outputs) {
-            auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
+            auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
             auto hash = hashString(htSHA256, "fixed:out:"
                 + dof.ca.printMethodAlgo() + ":"
                 + dof.ca.hash.to_string(Base16, false) + ":"
@@ -663,7 +663,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         [](const DerivationType::Impure &) -> DrvHash::Kind {
             assert(false);
         }
-    }, drv.type().raw());
+    }, drv.type().raw);
 
     std::map<std::string, StringSet> inputs2;
     for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
@@ -720,10 +720,10 @@ StringSet BasicDerivation::outputNames() const
 DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
 {
     DerivationOutputsAndOptPaths outsAndOptPaths;
-    for (auto output : outputs)
+    for (auto & [outputName, output] : outputs)
         outsAndOptPaths.insert(std::make_pair(
-            output.first,
+            outputName,
-            std::make_pair(output.second, output.second.path(store, name, output.first))
+            std::make_pair(output, output.path(store, name, outputName))
             )
         );
     return outsAndOptPaths;
@@ -798,7 +798,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
                     << (doi.method.renderPrefix() + printHashType(doi.hashType))
                     << "impure";
             },
-        }, i.second.raw());
+        }, i.second.raw);
     }
     WorkerProto::write(store,
         WorkerProto::WriteConn { .to = out },
@@ -840,7 +840,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
     auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
     for (auto & [outputName, output] : drv.outputs) {
-        if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
+        if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) {
             auto h = get(hashModulo.hashes, outputName);
             if (!h)
                 throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)",
@@ -955,7 +955,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
             [&](const DerivationOutput::Impure &) {
                 /* Nothing to check */
             },
-        }, i.second.raw());
+        }, i.second.raw);
     }
 }
 
@@ -984,7 +984,7 @@ nlohmann::json DerivationOutput::toJSON(
             res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
             res["impure"] = true;
         },
-    }, raw());
+    }, raw);
     return res;
 }
 

src/libstore/derivations.hh

@@ -9,6 +9,7 @@
 #include "derived-path.hh"
 #include "sync.hh"
 #include "comparator.hh"
+#include "variant-wrapper.hh"
 
 #include <map>
 #include <variant>
@@ -21,21 +22,26 @@ class Store;
 /* Abstract syntax of derivations. */
 
 /**
+ * A single output of a BasicDerivation (and Derivation).
+ */
+struct DerivationOutput
+{
+    /**
      * The traditional non-fixed-output derivation type.
      */
-struct DerivationOutputInputAddressed
+    struct InputAddressed
-{
+    {
         StorePath path;
 
-    GENERATE_CMP(DerivationOutputInputAddressed, me->path);
+        GENERATE_CMP(InputAddressed, me->path);
-};
+    };
 
-/**
+    /**
      * Fixed-output derivations, whose output paths are content
      * addressed according to that fixed output.
      */
-struct DerivationOutputCAFixed
+    struct CAFixed
-{
+    {
         /**
          * Method and hash used for expected hash computation.
          *
@@ -51,16 +57,16 @@ struct DerivationOutputCAFixed
          */
         StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 
-    GENERATE_CMP(DerivationOutputCAFixed, me->ca);
+        GENERATE_CMP(CAFixed, me->ca);
-};
+    };
 
-/**
+    /**
      * Floating-output derivations, whose output paths are content
      * addressed, but not fixed, and so are dynamically calculated from
      * whatever the output ends up being.
      * */
-struct DerivationOutputCAFloating
+    struct CAFloating
-{
+    {
         /**
          * How the file system objects will be serialized for hashing
          */
@@ -71,23 +77,23 @@ struct DerivationOutputCAFloating
          */
         HashType hashType;
 
-    GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType);
+        GENERATE_CMP(CAFloating, me->method, me->hashType);
-};
+    };
 
-/**
+    /**
      * Input-addressed output which depends on a (CA) derivation whose hash
      * isn't known yet.
      */
-struct DerivationOutputDeferred {
+    struct Deferred {
-    GENERATE_CMP(DerivationOutputDeferred);
+        GENERATE_CMP(Deferred);
-};
+    };
 
-/**
+    /**
      * Impure output which is moved to a content-addressed location (like
      * CAFloating) but isn't registered as a realization.
      */
-struct DerivationOutputImpure
+    struct Impure
-{
+    {
         /**
          * How the file system objects will be serialized for hashing
          */
@@ -98,30 +104,27 @@ struct DerivationOutputImpure
          */
         HashType hashType;
 
-    GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType);
+        GENERATE_CMP(Impure, me->method, me->hashType);
-};
+    };
 
-typedef std::variant<
+    typedef std::variant<
-    DerivationOutputInputAddressed,
+        InputAddressed,
-    DerivationOutputCAFixed,
+        CAFixed,
-    DerivationOutputCAFloating,
+        CAFloating,
-    DerivationOutputDeferred,
+        Deferred,
-    DerivationOutputImpure
+        Impure
-> _DerivationOutputRaw;
+    > Raw;
 
-/**
+    Raw raw;
- * A single output of a BasicDerivation (and Derivation).
+
+    GENERATE_CMP(DerivationOutput, me->raw);
+
+    MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput);
+
+    /**
+     * Force choosing a variant
      */
-struct DerivationOutput : _DerivationOutputRaw
+    DerivationOutput() = delete;
-{
-    using Raw = _DerivationOutputRaw;
-    using Raw::Raw;
-
-    using InputAddressed = DerivationOutputInputAddressed;
-    using CAFixed = DerivationOutputCAFixed;
-    using CAFloating = DerivationOutputCAFloating;
-    using Deferred = DerivationOutputDeferred;
-    using Impure = DerivationOutputImpure;
 
     /**
      * \note when you use this function you should make sure that you're
@@ -131,10 +134,6 @@ struct DerivationOutput : _DerivationOutputRaw
      */
     std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 
-    inline const Raw & raw() const {
-        return static_cast<const Raw &>(*this);
-    }
-
     nlohmann::json toJSON(
         const Store & store,
         std::string_view drvName,
@@ -167,29 +166,32 @@ typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePat
  */
 typedef std::map<StorePath, StringSet> DerivationInputs;
 
-/**
+struct DerivationType {
+    /**
      * Input-addressed derivation types
      */
-struct DerivationType_InputAddressed {
+    struct InputAddressed {
         /**
          * True iff the derivation type can't be determined statically,
          * for instance because it (transitively) depends on a content-addressed
          * derivation.
         */
         bool deferred;
-};
 
-/**
+        GENERATE_CMP(InputAddressed, me->deferred);
+    };
+
+    /**
      * Content-addressed derivation types
      */
-struct DerivationType_ContentAddressed {
+    struct ContentAddressed {
         /**
          * Whether the derivation should be built safely inside a sandbox.
          */
         bool sandboxed;
         /**
          * Whether the derivation's outputs' content-addresses are "fixed"
-     * or "floating.
+         * or "floating".
          *
          *  - Fixed: content-addresses are written down as part of the
          *    derivation itself. If the outputs don't end up matching the
@@ -199,29 +201,36 @@ struct DerivationType_ContentAddressed {
          *    know them until we perform the build.
          */
         bool fixed;
-};
 
-/**
+        GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed);
+    };
+
+    /**
      * Impure derivation type
      *
      * This is similar at buil-time to the content addressed, not standboxed, not fixed
      * type, but has some restrictions on its usage.
      */
-struct DerivationType_Impure {
+    struct Impure {
-};
+        GENERATE_CMP(Impure);
+    };
 
-typedef std::variant<
+    typedef std::variant<
-    DerivationType_InputAddressed,
+        InputAddressed,
-    DerivationType_ContentAddressed,
+        ContentAddressed,
-    DerivationType_Impure
+        Impure
-> _DerivationTypeRaw;
+    > Raw;
 
-struct DerivationType : _DerivationTypeRaw {
+    Raw raw;
-    using Raw = _DerivationTypeRaw;
+
-    using Raw::Raw;
+    GENERATE_CMP(DerivationType, me->raw);
-    using InputAddressed = DerivationType_InputAddressed;
+
-    using ContentAddressed = DerivationType_ContentAddressed;
+    MAKE_WRAPPER_CONSTRUCTOR(DerivationType);
-    using Impure = DerivationType_Impure;
+
+    /**
+     * Force choosing a variant
+     */
+    DerivationType() = delete;
 
     /**
      * Do the outputs of the derivation have paths calculated from their
@@ -257,10 +266,6 @@ struct DerivationType : _DerivationTypeRaw {
      * closure, or if fixed output.
      */
     bool hasKnownOutputPaths() const;
-
-    inline const Raw & raw() const {
-        return static_cast<const Raw &>(*this);
-    }
 };
 
 struct BasicDerivation

src/libstore/downstream-placeholder.cc

@@ -39,16 +39,18 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
 }
 
 DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
-    const SingleDerivedPath::Built & b)
+    const SingleDerivedPath::Built & b,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     return std::visit(overloaded {
         [&](const SingleDerivedPath::Opaque & o) {
-            return DownstreamPlaceholder::unknownCaOutput(o.path, b.output);
+            return DownstreamPlaceholder::unknownCaOutput(o.path, b.output, xpSettings);
         },
         [&](const SingleDerivedPath::Built & b2) {
             return DownstreamPlaceholder::unknownDerivation(
-                DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2),
+                DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2, xpSettings),
-                b.output);
+                b.output,
+                xpSettings);
         },
     }, b.drvPath->raw());
 }

src/libstore/downstream-placeholder.hh

@@ -84,7 +84,8 @@ public:
      * `SingleDerivedPath::Built.drvPath` chain.
      */
     static DownstreamPlaceholder fromSingleDerivedPathBuilt(
-        const SingleDerivedPath::Built & built);
+        const SingleDerivedPath::Built & built,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 };
 
 }

src/libstore/filetransfer.cc

@@ -863,6 +863,8 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
             }
 
             chunk = std::move(state->data);
+            /* Reset state->data after the move, since we check data.empty() */
+            state->data = "";
 
             state->request.notify_one();
         }

src/libstore/misc.cc

@@ -88,7 +88,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv)
     auto out = drv.outputs.find("out");
     if (out == drv.outputs.end())
         return nullptr;
-    if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
+    if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second.raw)) {
         return &dof->ca;
     }
     return nullptr;
@@ -370,7 +370,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd,
             }
             return outputsOpt;
         },
-    }, bfd.outputs.raw());
+    }, bfd.outputs.raw);
 
     OutputPathMap outputs;
     for (auto & [outputName, outputPathOpt] : outputsOpt) {
@@ -418,7 +418,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd)
         [&](const OutputsSpec::Names & names) {
             return static_cast<StringSet>(names);
         },
-    }, bfd.outputs.raw());
+    }, bfd.outputs.raw);
     for (auto iter = outputMap.begin(); iter != outputMap.end();) {
         auto & outputName = iter->first;
         if (bfd.outputs.contains(outputName)) {
@@ -431,7 +431,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd)
     if (!outputsLeft.empty())
         throw Error("derivation '%s' does not have an outputs %s",
             store.printStorePath(drvPath),
-            concatStringsSep(", ", quoteStrings(std::get<OutputsSpec::Names>(bfd.outputs))));
+            concatStringsSep(", ", quoteStrings(std::get<OutputsSpec::Names>(bfd.outputs.raw))));
     return outputMap;
 }
 

src/libstore/outputs-spec.cc

@@ -17,7 +17,7 @@ bool OutputsSpec::contains(const std::string & outputName) const
         [&](const OutputsSpec::Names & outputNames) {
             return outputNames.count(outputName) > 0;
         },
-    }, raw());
+    }, raw);
 }
 
 static std::string outputSpecRegexStr =
@@ -49,7 +49,7 @@ OutputsSpec OutputsSpec::parse(std::string_view s)
     std::optional spec = parseOpt(s);
     if (!spec)
         throw Error("invalid outputs specifier '%s'", s);
-    return *spec;
+    return std::move(*spec);
 }
 
 
@@ -85,7 +85,7 @@ std::string OutputsSpec::to_string() const
         [&](const OutputsSpec::Names & outputNames) -> std::string {
             return concatStringsSep(",", outputNames);
         },
-    }, raw());
+    }, raw);
 }
 
 
@@ -98,7 +98,7 @@ std::string ExtendedOutputsSpec::to_string() const
         [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string {
             return "^" + outputSpec.to_string();
         },
-    }, raw());
+    }, raw);
 }
 
 
@@ -118,9 +118,9 @@ OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const
                     ret.insert(thoseNames.begin(), thoseNames.end());
                     return ret;
                 },
-            }, that.raw());
+            }, that.raw);
         },
-    }, raw());
+    }, raw);
 }
 
 
@@ -142,9 +142,9 @@ bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const
                             ret = false;
                     return ret;
                 },
-            }, raw());
+            }, raw);
         },
-    }, that.raw());
+    }, that.raw);
 }
 
 }
@@ -169,7 +169,7 @@ void adl_serializer<OutputsSpec>::to_json(json & json, OutputsSpec t) {
         [&](const OutputsSpec::Names & names) {
             json = names;
         },
-    }, t.raw());
+    }, t.raw);
 }
 
 
@@ -189,7 +189,7 @@ void adl_serializer<ExtendedOutputsSpec>::to_json(json & json, ExtendedOutputsSp
         [&](const ExtendedOutputsSpec::Explicit & e) {
             adl_serializer<OutputsSpec>::to_json(json, e);
         },
-    }, t.raw());
+    }, t.raw);
 }
 
 }

src/libstore/outputs-spec.hh

@@ -6,62 +6,57 @@
 #include <set>
 #include <variant>
 
+#include "comparator.hh"
 #include "json-impls.hh"
+#include "comparator.hh"
+#include "variant-wrapper.hh"
 
 namespace nix {
 
-/**
+struct OutputsSpec {
+    /**
      * A non-empty set of outputs, specified by name
      */
-struct OutputNames : std::set<std::string> {
+    struct Names : std::set<std::string> {
         using std::set<std::string>::set;
 
         /* These need to be "inherited manually" */
 
-    OutputNames(const std::set<std::string> & s)
+        Names(const std::set<std::string> & s)
             : std::set<std::string>(s)
         { assert(!empty()); }
 
         /**
          * Needs to be "inherited manually"
          */
-    OutputNames(std::set<std::string> && s)
+        Names(std::set<std::string> && s)
             : std::set<std::string>(s)
         { assert(!empty()); }
 
         /* This set should always be non-empty, so we delete this
            constructor in order make creating empty ones by mistake harder.
            */
-    OutputNames() = delete;
+        Names() = delete;
-};
+    };
 
-/**
+    /**
      * The set of all outputs, without needing to name them explicitly
      */
-struct AllOutputs : std::monostate { };
+    struct All : std::monostate { };
 
-typedef std::variant<AllOutputs, OutputNames> _OutputsSpecRaw;
+    typedef std::variant<All, Names> Raw;
 
-struct OutputsSpec : _OutputsSpecRaw {
+    Raw raw;
-    using Raw = _OutputsSpecRaw;
+
-    using Raw::Raw;
+    GENERATE_CMP(OutputsSpec, me->raw);
+
+    MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec);
 
     /**
      * Force choosing a variant
      */
     OutputsSpec() = delete;
 
-    using Names = OutputNames;
-    using All = AllOutputs;
-
-    inline const Raw & raw() const {
-        return static_cast<const Raw &>(*this);
-    }
-
-    inline Raw & raw() {
-        return static_cast<Raw &>(*this);
-    }
-
     bool contains(const std::string & output) const;
 
     /**
@@ -84,20 +79,22 @@ struct OutputsSpec : _OutputsSpecRaw {
     std::string to_string() const;
 };
 
-struct DefaultOutputs : std::monostate { };
+struct ExtendedOutputsSpec {
-
+    struct Default : std::monostate { };
-typedef std::variant<DefaultOutputs, OutputsSpec> _ExtendedOutputsSpecRaw;
-
-struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw {
-    using Raw = _ExtendedOutputsSpecRaw;
-    using Raw::Raw;
-
-    using Default = DefaultOutputs;
     using Explicit = OutputsSpec;
 
-    inline const Raw & raw() const {
+    typedef std::variant<Default, Explicit> Raw;
-        return static_cast<const Raw &>(*this);
+
-    }
+    Raw raw;
+
+    GENERATE_CMP(ExtendedOutputsSpec, me->raw);
+
+    MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec);
+
+    /**
+     * Force choosing a variant
+     */
+    ExtendedOutputsSpec() = delete;
 
     /**
      * Parse a string of the form 'prefix^output1,...outputN' or

src/libstore/path-with-outputs.cc

@@ -63,7 +63,7 @@ StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const
                             [&](const OutputsSpec::Names & outputs) {
                                 return static_cast<StringSet>(outputs);
                             },
-                        }, bfd.outputs.raw()),
+                        }, bfd.outputs.raw),
                     };
                 },
                 [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult {

src/libutil/args.cc

@@ -410,8 +410,8 @@ nlohmann::json MultiCommand::toJSON()
         auto cat = nlohmann::json::object();
         cat["id"] = command->category();
         cat["description"] = trim(categories[command->category()]);
-        j["category"] = std::move(cat);
         cat["experimental-feature"] = command->experimentalFeature();
+        j["category"] = std::move(cat);
         cmds[name] = std::move(j);
     }
 

src/libutil/logging.cc

@@ -67,7 +67,7 @@ public:
             case lvlWarn: c = '4'; break;
             case lvlNotice: case lvlInfo: c = '5'; break;
             case lvlTalkative: case lvlChatty: c = '6'; break;
-            case lvlDebug: case lvlVomit: c = '7';
+            case lvlDebug: case lvlVomit: c = '7'; break;
             default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum
             }
             prefix = std::string("<") + c + ">";

src/libutil/url.cc

@@ -44,7 +44,7 @@ ParsedURL parseURL(const std::string & url)
             .base = base,
             .scheme = scheme,
             .authority = authority,
-            .path = path,
+            .path = percentDecode(path),
             .query = decodeQuery(query),
             .fragment = percentDecode(std::string(fragment))
         };

src/libutil/variant-wrapper.hh

@@ -0,0 +1,30 @@
+#pragma once
+///@file
+
+// not used, but will be used by callers
+#include <variant>
+
+/**
+ * Force the default versions of all constructors (copy, move, copy
+ * assignment).
+ */
+#define FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \
+    CLASS_NAME(const CLASS_NAME &) = default; \
+    CLASS_NAME(CLASS_NAME &) = default; \
+    CLASS_NAME(CLASS_NAME &&) = default; \
+    \
+    CLASS_NAME & operator =(const CLASS_NAME &) = default; \
+    CLASS_NAME & operator =(CLASS_NAME &) = default;
+
+/**
+ * Make a wrapper constructor. All args are forwarded to the
+ * construction of the "raw" field. (Which we assume is the only one.)
+ *
+ * The moral equivalent of `using Raw::Raw;`
+ */
+#define MAKE_WRAPPER_CONSTRUCTOR(CLASS_NAME) \
+    FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \
+    \
+    CLASS_NAME(auto &&... arg) \
+        : raw(std::forward<decltype(arg)>(arg)...) \
+    { }

src/nix/app.cc

@@ -81,7 +81,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state)
                         .path = o.path,
                     };
                 },
-            }, c.raw()));
+            }, c.raw));
         }
 
         return UnresolvedApp{App {

src/nix/bundle.cc

@@ -80,7 +80,7 @@ struct CmdBundle : InstallableValueCommand
         auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
         InstallableFlake bundler{this,
-            evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec,
+            evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec),
             {"bundlers." + settings.thisSystem.get() + ".default",
              "defaultBundler." + settings.thisSystem.get()
             },

src/nix/develop.cc

@@ -547,7 +547,7 @@ struct CmdDevelop : Common, MixEnvironment
                 state,
                 std::move(nixpkgs),
                 "bashInteractive",
-                DefaultOutputs(),
+                ExtendedOutputsSpec::Default(),
                 Strings{},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 nixpkgsLockFlags);

src/nix/flake.cc

@@ -778,7 +778,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
         auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
 
         auto installable = InstallableFlake(nullptr,
-            evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
+            evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(),
             defaultTemplateAttrPaths,
             defaultTemplateAttrPathsPrefixes,
             lockFlags);

src/nix/main.cc

@@ -359,6 +359,7 @@ void mainWrapped(int argc, char * * argv)
         experimentalFeatureSettings.experimentalFeatures = {
             Xp::Flakes,
             Xp::FetchClosure,
+            Xp::DynamicDerivations,
         };
         evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"));

tests/dyn-drv/dep-built-drv.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
+
+clearStore
+
+expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported"

tests/dyn-drv/eval-outputOf.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+source ./common.sh
+
+# Without the dynamic-derivations XP feature, we don't have the builtin.
+nix --experimental-features 'nix-command' eval --impure  --expr \
+    'assert ! (builtins ? outputOf); ""'
+
+# Test that a string is required.
+#
+# We currently require a string to be passed, rather than a derivation
+# object that could be coerced to a string. We might liberalise this in
+# the future so it does work, but there are some design questions to
+# resolve first. Adding a test so we don't liberalise it by accident.
+expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
+    'builtins.outputOf (import ../dependencies.nix) "out"' \
+    | grepQuiet "value is a set while a string was expected"
+
+# Test that "DrvDeep" string contexts are not supported at this time
+#
+# Like the above, this is a restriction we could relax later.
+expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
+    'builtins.outputOf (import ../dependencies.nix).drvPath "out"' \
+    | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time"
+
+# Test using `builtins.outputOf` with static derivations
+testStaticHello () {
+    nix eval --impure --expr \
+        'with (import ./text-hashed-output.nix); let
+           a = hello.outPath;
+           b = builtins.outputOf (builtins.unsafeDiscardOutputDependency hello.drvPath) "out";
+         in builtins.trace a
+           (builtins.trace b
+             (assert a == b; null))'
+}
+
+# Test with a regular old input-addresed derivation
+#
+# `builtins.outputOf` works without ca-derivations and doesn't create a
+# placeholder but just returns the output path.
+testStaticHello
+
+# Test with content addressed derivation.
+NIX_TESTS_CA_BY_DEFAULT=1 testStaticHello
+
+# Test with derivation-producing derivation
+#
+# This is hardly different from the preceding cases, except that we're
+# only taking 1 outputOf out of 2 possible outputOfs. Note that
+# `.outPath` could be defined as `outputOf drvPath`, which is what we're
+# testing here. The other `outputOf` that we're not testing here is the
+# use of _dynamic_ derivations.
+nix eval --impure --expr \
+    'with (import ./text-hashed-output.nix); let
+       a = producingDrv.outPath;
+       b = builtins.outputOf (builtins.builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out";
+     in builtins.trace a
+       (builtins.trace b
+         (assert a == b; null))'
+
+# Test with unbuilt output of derivation-producing derivation.
+#
+# This function similar to `testStaticHello` used above, but instead of
+# checking the property on a constant derivation, we check it on a
+# derivation that's from another derivation's output (outPath).
+testDynamicHello () {
+    nix eval --impure --expr \
+        'with (import ./text-hashed-output.nix); let
+           a = builtins.outputOf producingDrv.outPath "out";
+           b = builtins.outputOf (builtins.outputOf (builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out") "out";
+         in builtins.trace a
+           (builtins.trace b
+             (assert a == b; null))'
+}
+
+# inner dynamic derivation is input-addressed
+testDynamicHello
+
+# inner dynamic derivation is content-addressed
+NIX_TESTS_CA_BY_DEFAULT=1 testDynamicHello

tests/dyn-drv/local.mk

@@ -1,7 +1,9 @@
 dyn-drv-tests := \
   $(d)/text-hashed-output.sh \
   $(d)/recursive-mod-json.sh \
-  $(d)/build-built-drv.sh
+  $(d)/build-built-drv.sh \
+  $(d)/eval-outputOf.sh \
+  $(d)/dep-built-drv.sh
 
 install-tests-groups += dyn-drv
 

tests/dyn-drv/recursive-mod-json.sh

@@ -3,6 +3,8 @@ source common.sh
 # FIXME
 if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
 
+export NIX_TESTS_CA_BY_DEFAULT=1
+
 enableFeatures 'recursive-nix'
 restartDaemon
 

tests/dyn-drv/text-hashed-output.nix

@@ -12,9 +12,6 @@ rec {
       mkdir -p $out
       echo "Hello World" > $out/hello
     '';
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
   };
   producingDrv = mkDerivation {
     name = "hello.drv";
@@ -26,4 +23,11 @@ rec {
     outputHashMode = "text";
     outputHashAlgo = "sha256";
   };
+  wrapper = mkDerivation {
+    name = "use-dynamic-drv-in-non-dynamic-drv";
+    buildCommand = ''
+      echo "Copying the output of the dynamic derivation"
+      cp -r ${builtins.outputOf producingDrv.outPath "out"} $out
+    '';
+  };
 }

tests/repl.sh

@@ -56,11 +56,17 @@ testRepl
 # Same thing (kind-of), but with a remote store.
 testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR"
 
-testReplResponse () {
+# Remove ANSI escape sequences. They can prevent grep from finding a match.
+stripColors () {
+    sed -E 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g'
+}
+
+testReplResponseGeneral () {
+    local grepMode="$1"; shift
     local commands="$1"; shift
     local expectedResponse="$1"; shift
-    local response="$(nix repl "$@" <<< "$commands")"
+    local response="$(nix repl "$@" <<< "$commands" | stripColors)"
-    echo "$response" | grepQuiet -s "$expectedResponse" \
+    echo "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \
       || fail "repl command set:
 
 $commands
@@ -71,7 +77,16 @@ $expectedResponse
 
 but with:
 
-$response"
+$response
+"
+}
+
+testReplResponse () {
+    testReplResponseGeneral --basic-regexp "$@"
+}
+
+testReplResponseNoRegex () {
+    testReplResponseGeneral --fixed-strings "$@"
 }
 
 # :a uses the newest version of a symbol
@@ -85,9 +100,9 @@ testReplResponse '
 # note the escaped \,
 #    \\
 # because the second argument is a regex
-testReplResponse '
+testReplResponseNoRegex '
 "$" + "{hi}"
-' '"\\${hi}"'
+' '"\${hi}"'
 
 testReplResponse '
 drvPath
@@ -133,3 +148,34 @@ echo "changingThing"
 ) | nix repl ./flake --experimental-features 'flakes repl-flake')
 echo "$replResult" | grepQuiet -s beforeChange
 echo "$replResult" | grepQuiet -s afterChange
+
+# Test recursive printing and formatting
+# Normal output should print attributes in lexicographical order non-recursively
+testReplResponseNoRegex '
+{ a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; }
+' '{ a = { ... }; l = [ ... ]; n = 1234; s = "string"; x = { ... }; }'
+
+# Same for lists, but order is preserved
+testReplResponseNoRegex '
+[ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ]
+' '[ 42 1 "thingy" { ... } [ ... ] ]'
+
+# Same for let expressions
+testReplResponseNoRegex '
+let x = { y = { a = 1; }; inherit x; }; in x
+' '{ x = { ... }; y = { ... }; }'
+
+# The :p command should recursively print sets, but prevent infinite recursion
+testReplResponseNoRegex '
+:p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; }
+' '{ a = { b = 2; }; n = 1234; s = "string"; x = { y = { z = { y = «repeated»; }; }; }; }'
+
+# Same for lists
+testReplResponseNoRegex '
+:p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ]
+' '[ 42 1 "thingy" { a = 1; b = { a = 1; b = «repeated»; }; } [ 1 2 3 ] ]'
+
+# Same for let expressions
+testReplResponseNoRegex '
+:p let x = { y = { a = 1; }; inherit x; }; in x
+' '{ x = { x = «repeated»; y = { a = 1; }; }; y = «repeated»; }'