Allow dynamic derivation deps in inputDrvs

We use the same nested map representation we used for goals, again in
order to save space. We might someday want to combine with `inputDrvs`,
by doing `V = bool` instead of `V = std::set<OutputName>`, but we are
not doing that yet for sake of a smaller diff.

The ATerm format for Derivations also needs to be extended, in addition
to the in-memory format. To accomodate this, we added a new basic
versioning scheme, so old versions of Nix will get nice errors. (And
going forward, if the ATerm format changes again the errors will be even
better.)

`parsedStrings`, an internal function used as part of parsing
derivations in A-Term format, used to consume the final `]` but expect
the initial `[` to already be consumed. This made for what looked like
unbalanced brackets at callsites, which was confusing. Now it consumes
both which is hopefully less confusing.

As part of testing, we also created a unit test for the A-Term format for
regular non-experimental derivations too.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>

Apply suggestions from code review

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>

doc/manual/src/protocols/derivation-aterm.md

@@ -2,8 +2,18 @@
 
 For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format.
 
-Derivations are serialised in the following format:
+Derivations are serialised in one of the following formats:
 
-```
+- ```
-Derive(...)
+  Derive(...)
-```
+  ```
+
+  For all stable derivations.
+
+- ```
+  DrvWithVersion(<version-string>, ...)
+  ```
+
+  The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md):
+
+  - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

perl/lib/Nix/Store.xs

@@ -324,7 +324,7 @@ SV * derivationFromPath(char * drvPath)
             hv_stores(hash, "outputs", newRV((SV *) outputs));
 
             AV * inputDrvs = newAV();
-            for (auto & i : drv.inputDrvs)
+            for (auto & i : drv.inputDrvs.map)
                 av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second
             hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
 

src/build-remote/build-remote.cc

@@ -314,7 +314,7 @@ connected:
             //
             // 2. Changing the `inputSrcs` set changes the associated
             //    output ids, which break CA derivations
-            if (!drv.inputDrvs.empty())
+            if (!drv.inputDrvs.map.empty())
                 drv.inputSrcs = store->parseStorePathSet(inputs);
             optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
             auto & result = *optResult;

src/libcmd/built-path.cc

@@ -58,6 +58,28 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
+SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const
+{
+    return SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(drvPath->discardOutputPath()),
+        .output = output.first,
+    };
+}
+
+SingleDerivedPath SingleBuiltPath::discardOutputPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
+                return p;
+            },
+            [](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
+                return b.discardOutputPath();
+            },
+        }, raw()
+    );
+}
+
 nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
 {
     nlohmann::json res;

src/libcmd/built-path.hh

@@ -9,6 +9,8 @@ struct SingleBuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::pair<std::string, StorePath> output;
 
+    SingleDerivedPathBuilt discardOutputPath() const;
+
     std::string to_string(const Store & store) const;
     static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const Store & store) const;
@@ -34,6 +36,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
 
     StorePath outPath() const;
 
+    SingleDerivedPath discardOutputPath() const;
+
     static SingleBuiltPath parse(const Store & store, std::string_view);
     nlohmann::json toJSON(const Store & store) const;
 };

src/libexpr/primops.cc

@@ -1252,15 +1252,13 @@ drvName, Bindings * attrs, Value & v)
                 state.store->computeFSClosure(d.drvPath, refs);
                 for (auto & j : refs) {
                     drv.inputSrcs.insert(j);
-                    if (j.isDerivation())
+                    if (j.isDerivation()) {
-                        drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
+                        drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames();
+                    }
                 }
             },
             [&](const NixStringContextElem::Built & b) {
-                if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath))
+                drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
-                    drv.inputDrvs[p->path].insert(b.output);
-                else
-                    throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported");
             },
             [&](const NixStringContextElem::Opaque & o) {
                 drv.inputSrcs.insert(o.path);

src/libstore/build/derivation-goal.cc

@@ -350,25 +350,37 @@ void DerivationGoal::gaveUpOnSubstitution()
 
     /* The inputs must be built before we can build this goal. */
     inputDrvOutputs.clear();
-    if (useDerivation)
+    if (useDerivation) {
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> addWaiteeDerivedPath;
+
+        addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            if (!inputNode.value.empty())
+                addWaitee(worker.makeGoal(
+                    DerivedPath::Built {
+                        .drvPath = inputDrv,
+                        .outputs = inputNode.value,
+                    },
+                    buildMode == bmRepair ? bmRepair : bmNormal));
+            for (const auto & [outputName, childNode] : inputNode.childMap)
+                addWaiteeDerivedPath(
+                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                    childNode);
+        };
+
+        for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
             /* Ensure that pure, non-fixed-output derivations don't
                depend on impure derivations. */
             if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
-                auto inputDrv = worker.evalStore.readDerivation(i.first);
+                auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
                 if (!inputDrv.type().isPure())
                     throw Error("pure derivation '%s' depends on impure derivation '%s'",
                         worker.store.printStorePath(drvPath),
-                        worker.store.printStorePath(i.first));
+                        worker.store.printStorePath(inputDrvPath));
             }
 
-            addWaitee(worker.makeGoal(
+            addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
-                DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(i.first),
-                    .outputs = i.second,
-                },
-                buildMode == bmRepair ? bmRepair : bmNormal));
         }
+    }
 
     /* Copy the input sources from the eval store to the build
        store. */
@@ -501,7 +513,7 @@ void DerivationGoal::inputsRealised()
                 return ia.deferred;
             },
             [&](const DerivationType::ContentAddressed & ca) {
-                return !fullDrv.inputDrvs.empty() && (
+                return !fullDrv.inputDrvs.map.empty() && (
                     ca.fixed
                     /* Can optionally resolve if fixed, which is good
                        for avoiding unnecessary rebuilds. */
@@ -515,7 +527,7 @@ void DerivationGoal::inputsRealised()
             }
         }, drvType.raw);
 
-        if (resolveDrv && !fullDrv.inputDrvs.empty()) {
+        if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
             experimentalFeatureSettings.require(Xp::CaDerivations);
 
             /* We are be able to resolve this derivation based on the
@@ -552,11 +564,13 @@ void DerivationGoal::inputsRealised()
             return;
         }
 
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
+        std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
+
+        accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            for (auto & j : wantedDepOutputs) {
+            auto getOutput = [&](const std::string & outputName) {
                 /* TODO (impure derivations-induced tech debt):
                    Tracking input derivation outputs statefully through the
                    goals is error prone and has led to bugs.
@@ -568,21 +582,30 @@ void DerivationGoal::inputsRealised()
                    a representation in the store, which is a usability problem
                    in itself. When implementing this logic entirely with lookups
                    make sure that they're cached. */
-                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) {
-                    worker.store.computeFSClosure(*outPath, inputPaths);
+                    return *outPath;
                 }
                 else {
                     auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
-                    auto outMapPath = outMap.find(j);
+                    auto outMapPath = outMap.find(outputName);
                     if (outMapPath == outMap.end()) {
                         throw Error(
                             "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                            worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath));
                     }
-                    worker.store.computeFSClosure(outMapPath->second, inputPaths);
+                    return outMapPath->second;
                 }
-            }
+            };
-        }
+
+            for (auto & outputName : inputNode.value)
+                worker.store.computeFSClosure(getOutput(outputName), inputPaths);
+
+            for (auto & [outputName, childNode] : inputNode.childMap)
+                accumInputPaths(getOutput(outputName), childNode);
+        };
+
+        for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map)
+            accumInputPaths(depDrvPath, depNode);
     }
 
     /* Second, the input sources. */
@@ -1475,22 +1498,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
     if (!useDerivation) return;
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    auto * dg = tryGetConcreteDrvGoal(waitee);
+    std::optional info = tryGetConcreteDrvGoal(waitee);
-    if (!dg) return;
+    if (!info) return;
+    const auto & [dg, drvReq] = *info;
 
-    auto outputs = fullDrv.inputDrvs.find(dg->drvPath);
+    auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get());
-    if (outputs == fullDrv.inputDrvs.end()) return;
+    if (!nodeP) return;
+    auto & outputs = nodeP->value;
 
-    for (auto & outputName : outputs->second) {
+    for (auto & outputName : outputs) {
-        auto buildResult = dg->getBuildResult(DerivedPath::Built {
+        auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(dg->drvPath),
+            .drvPath = makeConstantStorePathRef(dg.get().drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {
             auto i = buildResult.builtOutputs.find(outputName);
             if (i != buildResult.builtOutputs.end())
                 inputDrvOutputs.insert_or_assign(
-                    { dg->drvPath, outputName },
+                    { dg.get().drvPath, outputName },
                     i->second.outPath);
         }
     }

src/libstore/build/worker.cc

@@ -594,11 +594,14 @@ GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
     return subGoal;
 }
 
-const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee)
+std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
 {
     auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
-    if (!odg) return nullptr;
+    if (!odg) return std::nullopt;
-    return &*odg->concreteDrvGoal;
+    return {{
+        std::cref(*odg->concreteDrvGoal),
+        std::cref(*odg->drvReq),
+    }};
 }
 
 }

src/libstore/build/worker.hh

@@ -49,7 +49,8 @@ typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
  * we have made the function, written in `worker.cc` where all the goal
  * types are visible, and use it instead.
  */
-const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee);
+
+std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
 
 /**
  * A mapping used to remember for each child process to what goal it

src/libstore/derivations.cc

@@ -136,7 +136,7 @@ StorePath writeDerivation(Store & store,
     const Derivation & drv, RepairFlag repair, bool readOnly)
 {
     auto references = drv.inputSrcs;
-    for (auto & i : drv.inputDrvs)
+    for (auto & i : drv.inputDrvs.map)
         references.insert(i.first);
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
@@ -208,22 +208,25 @@ static bool endOfList(std::istream & str)
 static StringSet parseStrings(std::istream & str, bool arePaths)
 {
     StringSet res;
+    expect(str, "[");
     while (!endOfList(str))
         res.insert(arePaths ? parsePath(str) : parseString(str));
     return res;
 }
 
 
-static DerivationOutput parseDerivationOutput(const Store & store,
+static DerivationOutput parseDerivationOutput(
-    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
+    const Store & store,
+    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     if (hashAlgo != "") {
         ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
         if (method == TextIngestionMethod {})
-            experimentalFeatureSettings.require(Xp::DynamicDerivations);
+            xpSettings.require(Xp::DynamicDerivations);
         const auto hashType = parseHashType(hashAlgo);
         if (hashS == "impure") {
-            experimentalFeatureSettings.require(Xp::ImpureDerivations);
+            xpSettings.require(Xp::ImpureDerivations);
             if (pathS != "")
                 throw FormatError("impure derivation output should not specify output path");
             return DerivationOutput::Impure {
@@ -240,7 +243,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
                 },
             };
         } else {
-            experimentalFeatureSettings.require(Xp::CaDerivations);
+            xpSettings.require(Xp::CaDerivations);
             if (pathS != "")
                 throw FormatError("content-addressed derivation output should not specify output path");
             return DerivationOutput::CAFloating {
@@ -259,29 +262,116 @@ static DerivationOutput parseDerivationOutput(const Store & store,
     }
 }
 
-static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+static DerivationOutput parseDerivationOutput(
+    const Store & store, std::istringstream & str,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
     expect(str, ","); const auto hashAlgo = parseString(str);
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");
 
-    return parseDerivationOutput(store, pathS, hashAlgo, hash);
+    return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings);
+}
+
+/**
+ * All ATerm Derivation format versions currently known.
+ *
+ * Unknown versions are rejected at the parsing stage.
+ */
+enum struct DerivationATermVersion {
+    /**
+     * Older unversioned form
+     */
+    Traditional,
+
+    /**
+     * Newer versioned form; only this version so far.
+     */
+    DynamicDerivations,
+};
+
+static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
+    const Store & store,
+    std::istringstream & str,
+    DerivationATermVersion version)
+{
+    DerivedPathMap<StringSet>::ChildNode node;
+
+    auto parseNonDynamic = [&]() {
+        node.value = parseStrings(str, false);
+    };
+
+    // Older derivation should never use new form, but newer
+    // derivaiton can use old form.
+    switch (version) {
+    case DerivationATermVersion::Traditional:
+        parseNonDynamic();
+        break;
+    case DerivationATermVersion::DynamicDerivations:
+        switch (str.peek()) {
+        case '[':
+            parseNonDynamic();
+            break;
+        case '(':
+            expect(str, "(");
+            node.value = parseStrings(str, false);
+            expect(str, ",[");
+            while (!endOfList(str)) {
+                expect(str, "(");
+                auto outputName = parseString(str);
+                expect(str, ",");
+                node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version));
+                expect(str, ")");
+            }
+            expect(str, ")");
+            break;
+        default:
+            throw FormatError("invalid inputDrvs entry in derivation");
+        }
+        break;
+    default:
+        // invalid format, not a parse error but internal error
+        assert(false);
+    }
+    return node;
 }
 
 
-Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
+Derivation parseDerivation(
+    const Store & store, std::string && s, std::string_view name,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     Derivation drv;
     drv.name = name;
 
     std::istringstream str(std::move(s));
-    expect(str, "Derive([");
+    expect(str, "D");
+    DerivationATermVersion version;
+    switch (str.peek()) {
+    case 'e':
+        expect(str, "erive(");
+        version = DerivationATermVersion::Traditional;
+        break;
+    case 'r':
+        expect(str, "rvWithVersion(");
+        auto versionS = parseString(str);
+        if (versionS == "xp-dyn-drv") {
+            // Only verison we have so far
+            version = DerivationATermVersion::DynamicDerivations;
+            xpSettings.require(Xp::DynamicDerivations);
+        } else {
+            throw FormatError("Unknown derivation ATerm format version '%s'", versionS);
+        }
+        expect(str, ",");
+        break;
+    }
 
     /* Parse the list of outputs. */
+    expect(str, "[");
     while (!endOfList(str)) {
         expect(str, "("); std::string id = parseString(str);
-        auto output = parseDerivationOutput(store, str);
+        auto output = parseDerivationOutput(store, str, xpSettings);
         drv.outputs.emplace(std::move(id), std::move(output));
     }
 
@@ -290,12 +380,12 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
     while (!endOfList(str)) {
         expect(str, "(");
         Path drvPath = parsePath(str);
-        expect(str, ",[");
+        expect(str, ",");
-        drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false));
+        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version));
         expect(str, ")");
     }
 
-    expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
+    expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
     expect(str, ","); drv.platform = parseString(str);
     expect(str, ","); drv.builder = parseString(str);
 
@@ -379,14 +469,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt
 }
 
 
+static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
+{
+    s += ',';
+    if (node.childMap.empty()) {
+        printUnquotedStrings(s, node.value.begin(), node.value.end());
+    } else {
+        s += "(";
+        printUnquotedStrings(s, node.value.begin(), node.value.end());
+        s += ",[";
+        bool first = true;
+        for (auto & [outputName, childNode] : node.childMap) {
+            if (first) first = false; else s += ',';
+            s += '('; printUnquotedString(s, outputName);
+            unparseDerivedPathMapNode(store, s, childNode);
+            s += ')';
+        }
+        s += "])";
+    }
+}
+
+
+/**
+ * Does the derivation have a dependency on the output of a dynamic
+ * derivation?
+ *
+ * In other words, does it on the output of derivation that is itself an
+ * ouput of a derivation? This corresponds to a dependency that is an
+ * inductive derived path with more than one layer of
+ * `DerivedPath::Built`.
+ */
+static bool hasDynamicDrvDep(const Derivation & drv)
+{
+    return
+        std::find_if(
+            drv.inputDrvs.map.begin(),
+            drv.inputDrvs.map.end(),
+            [](auto & kv) { return !kv.second.childMap.empty(); })
+        != drv.inputDrvs.map.end();
+}
+
+
 std::string Derivation::unparse(const Store & store, bool maskOutputs,
-    std::map<std::string, StringSet> * actualInputs) const
+    DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
 {
     std::string s;
     s.reserve(65536);
-    s += "Derive([";
+
+    /* Use older unversioned form if possible, for wider compat. Use
+       newer form only if we need it, which we do for
+       `Xp::DynamicDerivations`. */
+    if (hasDynamicDrvDep(*this)) {
+        s += "DrvWithVersion(";
+        // Only version we have so far
+        printUnquotedString(s, "xp-dyn-drv");
+        s += ",";
+    } else {
+        s += "Derive(";
+    }
 
     bool first = true;
+    s += "[";
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
@@ -424,17 +567,17 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
     s += "],[";
     first = true;
     if (actualInputs) {
-        for (auto & i : *actualInputs) {
+        for (auto & [drvHashModulo, childMap] : *actualInputs) {
             if (first) first = false; else s += ',';
-            s += '('; printUnquotedString(s, i.first);
+            s += '('; printUnquotedString(s, drvHashModulo);
-            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
+            unparseDerivedPathMapNode(store, s, childMap);
             s += ')';
         }
     } else {
-        for (auto & i : inputDrvs) {
+        for (auto & [drvPath, childMap] : inputDrvs.map) {
             if (first) first = false; else s += ',';
-            s += '('; printUnquotedString(s, store.printStorePath(i.first));
+            s += '('; printUnquotedString(s, store.printStorePath(drvPath));
-            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
+            unparseDerivedPathMapNode(store, s, childMap);
             s += ')';
         }
     }
@@ -668,18 +811,16 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         }
     }, drv.type().raw);
 
-    std::map<std::string, StringSet> inputs2;
+    DerivedPathMap<StringSet>::ChildNode::Map inputs2;
-    for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
+    for (auto & [drvPath, node] : drv.inputDrvs.map) {
-        // Avoid lambda capture restriction with standard / Clang
-        auto & inputOutputs = inputOutputs0;
         const auto & res = pathDerivationModulo(store, drvPath);
         if (res.kind == DrvHash::Kind::Deferred)
             kind = DrvHash::Kind::Deferred;
-        for (auto & outputName : inputOutputs) {
+        for (auto & outputName : node.value) {
             const auto h = get(res.hashes, outputName);
             if (!h)
                 throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
-            inputs2[h->to_string(Base16, false)].insert(outputName);
+            inputs2[h->to_string(Base16, false)].value.insert(outputName);
         }
     }
 
@@ -709,7 +850,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
     const auto hashAlgo = readString(in);
     const auto hash = readString(in);
 
-    return parseDerivationOutput(store, pathS, hashAlgo, hash);
+    return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings);
 }
 
 StringSet BasicDerivation::outputNames() const
@@ -824,6 +965,8 @@ std::string hashPlaceholder(const OutputNameView outputName)
 
 static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
 {
+    debug("Rewriting the derivation");
+
     for (auto & rewrite : rewrites) {
         debug("rewriting %s as %s", rewrite.first, rewrite.second);
     }
@@ -862,14 +1005,70 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
 {
     std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
-    for (auto & input : inputDrvs)
+    std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
-        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
+    accum = [&](auto & inputDrv, auto & node) {
-            if (outputPath)
+        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) {
-                inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
+            if (outputPath) {
+                inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
+                if (auto p = get(node.childMap, outputName))
+                    accum(*outputPath, *p);
+            }
+        }
+    };
+
+    for (auto & [inputDrv, node] : inputDrvs.map)
+        accum(inputDrv, node);
 
     return tryResolve(store, inputDrvOutputs);
 }
 
+static bool tryResolveInput(
+    Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
+    const DownstreamPlaceholder * placeholderOpt,
+    const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode,
+    const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs)
+{
+    auto getOutput = [&](const std::string & outputName) {
+        auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName });
+        if (!actualPathOpt)
+            warn("output %s of input %s missing, aborting the resolving",
+                outputName,
+                store.printStorePath(inputDrv)
+            );
+        return actualPathOpt;
+    };
+
+    auto getPlaceholder = [&](const std::string & outputName) {
+        return placeholderOpt
+            ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName)
+            : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName);
+    };
+
+    for (auto & outputName : inputNode.value) {
+        auto actualPathOpt = getOutput(outputName);
+        if (!actualPathOpt) return false;
+        auto actualPath = *actualPathOpt;
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+            inputRewrites.emplace(
+                getPlaceholder(outputName).render(),
+                store.printStorePath(actualPath));
+        }
+        inputSrcs.insert(std::move(actualPath));
+    }
+
+    for (auto & [outputName, childNode] : inputNode.childMap) {
+        auto actualPathOpt = getOutput(outputName);
+        if (!actualPathOpt) return false;
+        auto actualPath = *actualPathOpt;
+        auto nextPlaceholder = getPlaceholder(outputName);
+        if (!tryResolveInput(store, inputSrcs, inputRewrites,
+            &nextPlaceholder, actualPath, childNode,
+            inputDrvOutputs))
+            return false;
+    }
+    return true;
+}
+
 std::optional<BasicDerivation> Derivation::tryResolve(
     Store & store,
     const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
@@ -879,23 +1078,10 @@ std::optional<BasicDerivation> Derivation::tryResolve(
     // Input paths that we'll want to rewrite in the derivation
     StringMap inputRewrites;
 
-    for (auto & [inputDrv, inputOutputs] : inputDrvs) {
+    for (auto & [inputDrv, inputNode] : inputDrvs.map)
-        for (auto & outputName : inputOutputs) {
+        if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites,
-            if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
+            nullptr, inputDrv, inputNode, inputDrvOutputs))
-                if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+            return std::nullopt;
-                    inputRewrites.emplace(
-                        DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
-                        store.printStorePath(*actualPath));
-                }
-                resolved.inputSrcs.insert(*actualPath);
-            } else {
-                warn("output '%s' of input '%s' missing, aborting the resolving",
-                    outputName,
-                    store.printStorePath(inputDrv));
-                return {};
-            }
-        }
-    }
 
     rewriteDerivation(store, resolved, inputRewrites);
 
@@ -1084,10 +1270,25 @@ nlohmann::json Derivation::toJSON(const Store & store) const
     }
 
     {
-        auto& inputDrvsObj = res["inputDrvs"];
+        std::function<nlohmann::json(const DerivedPathMap<StringSet>::ChildNode &)> doInput;
-        inputDrvsObj = nlohmann::json ::object();
+        doInput = [&](const auto & inputNode) {
-        for (auto & input : inputDrvs)
+            auto value = nlohmann::json::object();
-            inputDrvsObj[store.printStorePath(input.first)] = input.second;
+            value["outputs"] = inputNode.value;
+            {
+                auto next = nlohmann::json::object();
+                for (auto & [outputId, childNode] : inputNode.childMap)
+                    next[outputId] = doInput(childNode);
+                value["dynamicOutputs"] = std::move(next);
+            }
+            return value;
+        };
+        {
+            auto& inputDrvsObj = res["inputDrvs"];
+            inputDrvsObj = nlohmann::json::object();
+            for (auto & [inputDrv, inputNode] : inputDrvs.map) {
+                inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode);
+            }
+        }
     }
 
     res["system"] = platform;
@@ -1101,7 +1302,8 @@ nlohmann::json Derivation::toJSON(const Store & store) const
 
 Derivation Derivation::fromJSON(
     const Store & store,
-    const nlohmann::json & json)
+    const nlohmann::json & json,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     using nlohmann::detail::value_t;
 
@@ -1133,12 +1335,21 @@ Derivation Derivation::fromJSON(
     }
 
     try {
+        std::function<DerivedPathMap<StringSet>::ChildNode(const nlohmann::json &)> doInput;
+        doInput = [&](const auto & json) {
+            DerivedPathMap<StringSet>::ChildNode node;
+            node.value = static_cast<const StringSet &>(
+                ensureType(valueAt(json, "outputs"), value_t::array));
+            for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) {
+                xpSettings.require(Xp::DynamicDerivations);
+                node.childMap[outputId] = doInput(childNode);
+            }
+            return node;
+        };
         auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object);
-        for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) {
+        for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
-            ensureType(inputOutputs, value_t::array);
+            res.inputDrvs.map[store.parseStorePath(inputDrvPath)] =
-            res.inputDrvs[store.parseStorePath(inputDrvPath)] =
+                doInput(inputOutputs);
-                static_cast<const StringSet &>(inputOutputs);
-        }
     } catch (Error & e) {
         e.addTrace({}, "while reading key 'inputDrvs'");
         throw;

src/libstore/derivations.hh

@@ -6,7 +6,7 @@
 #include "hash.hh"
 #include "content-address.hh"
 #include "repair-flag.hh"
-#include "derived-path.hh"
+#include "derived-path-map.hh"
 #include "sync.hh"
 #include "comparator.hh"
 #include "variant-wrapper.hh"
@@ -323,13 +323,13 @@ struct Derivation : BasicDerivation
     /**
      * inputs that are sub-derivations
      */
-    DerivationInputs inputDrvs;
+    DerivedPathMap<std::set<OutputName>> inputDrvs;
 
     /**
      * Print a derivation.
      */
     std::string unparse(const Store & store, bool maskOutputs,
-        std::map<std::string, StringSet> * actualInputs = nullptr) const;
+        DerivedPathMap<StringSet>::ChildNode::Map * actualInputs = nullptr) const;
 
     /**
      * Return the underlying basic derivation but with these changes:
@@ -368,7 +368,8 @@ struct Derivation : BasicDerivation
     nlohmann::json toJSON(const Store & store) const;
     static Derivation fromJSON(
         const Store & store,
-        const nlohmann::json & json);
+        const nlohmann::json & json,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     GENERATE_CMP(Derivation,
         static_cast<const BasicDerivation &>(*me),
@@ -389,7 +390,11 @@ StorePath writeDerivation(Store & store,
 /**
  * Read a derivation from a file.
  */
-Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
+Derivation parseDerivation(
+    const Store & store,
+    std::string && s,
+    std::string_view name,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
 /**
  * \todo Remove.

src/libstore/derived-path-map.cc

@@ -21,6 +21,32 @@ typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const Sing
     return initIter(k);
 }
 
+template<typename V>
+typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const SingleDerivedPath & k)
+{
+    std::function<ChildNode *(const SingleDerivedPath & )> initIter;
+    initIter = [&](const auto & k) {
+        return std::visit(overloaded {
+            [&](const SingleDerivedPath::Opaque & bo) {
+                auto it = map.find(bo.path);
+                return it != map.end()
+                    ? &it->second
+                    : nullptr;
+            },
+            [&](const SingleDerivedPath::Built & bfd) {
+                auto * n = initIter(*bfd.drvPath);
+                if (!n) return (ChildNode *)nullptr;
+
+                auto it = n->childMap.find(bfd.output);
+                return it != n->childMap.end()
+                    ? &it->second
+                    : nullptr;
+            },
+        }, k.raw());
+    };
+    return initIter(k);
+}
+
 }
 
 // instantiations
@@ -30,4 +56,17 @@ namespace nix {
 
 template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
 
-}
+GENERATE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>::ChildNode,
+    me->value,
+    me->childMap);
+
+GENERATE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>,
+    me->map);
+
+template struct DerivedPathMap<std::set<std::string>>;
+
+};

src/libstore/derived-path-map.hh

@@ -48,6 +48,8 @@ struct DerivedPathMap {
          * The map of the root node.
          */
         Map childMap;
+
+        DECLARE_CMP(ChildNode);
     };
 
     /**
@@ -60,6 +62,8 @@ struct DerivedPathMap {
      */
     Map map;
 
+    DECLARE_CMP(DerivedPathMap);
+
     /**
      * Find the node for `k`, creating it if needed.
      *
@@ -68,6 +72,27 @@ struct DerivedPathMap {
      * by changing this node.
      */
     ChildNode & ensureSlot(const SingleDerivedPath & k);
+
+    /**
+     * Like `ensureSlot` but does not create the slot if it doesn't exist.
+     *
+     * Read the entire description of `ensureSlot` to understand an
+     * important caveat here that "have slot" does *not* imply "key is
+     * set in map". To ensure a key is set one would need to get the
+     * child node (with `findSlot` or `ensureSlot`) *and* check the
+     * `ChildNode::value`.
+     */
+    ChildNode * findSlot(const SingleDerivedPath & k);
 };
 
+
+DECLARE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>::,
+    DerivedPathMap<std::set<std::string>>);
+DECLARE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>::ChildNode::,
+    DerivedPathMap<std::set<std::string>>::ChildNode);
+
 }

src/libstore/misc.cc

@@ -125,14 +125,26 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
 
     std::function<void(DerivedPath)> doPath;
 
+    std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> enqueueDerivedPaths;
+
+    enqueueDerivedPaths = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+        if (!inputNode.value.empty())
+            pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value }));
+        for (const auto & [outputName, childNode] : inputNode.childMap)
+            enqueueDerivedPaths(
+                make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                childNode);
+    };
+
     auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
         {
             auto state(state_.lock());
             state->willBuild.insert(drvPath);
         }
 
-        for (auto & i : drv.inputDrvs)
+        for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) {
-            pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second }));
+            enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode);
+        }
     };
 
     auto checkOutput = [&](
@@ -322,24 +334,41 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
 {
     std::set<Realisation> inputRealisations;
 
-    for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
+    std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
-        const auto outputHashes =
+
-            staticOutputHashes(store, store.readDerivation(inputDrv));
+    accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
-        for (const auto & outputName : outputNames) {
+        if (!inputNode.value.empty()) {
-            auto outputHash = get(outputHashes, outputName);
+            auto outputHashes =
-            if (!outputHash)
+                staticOutputHashes(store, store.readDerivation(inputDrv));
-                throw Error(
+            for (const auto & outputName : inputNode.value) {
-                    "output '%s' of derivation '%s' isn't realised", outputName,
+                auto outputHash = get(outputHashes, outputName);
-                    store.printStorePath(inputDrv));
+                if (!outputHash)
-            auto thisRealisation = store.queryRealisation(
+                    throw Error(
-                DrvOutput{*outputHash, outputName});
+                        "output '%s' of derivation '%s' isn't realised", outputName,
-            if (!thisRealisation)
+                        store.printStorePath(inputDrv));
-                throw Error(
+                auto thisRealisation = store.queryRealisation(
-                    "output '%s' of derivation '%s' isn't built", outputName,
+                    DrvOutput{*outputHash, outputName});
-                    store.printStorePath(inputDrv));
+                if (!thisRealisation)
-            inputRealisations.insert(*thisRealisation);
+                    throw Error(
+                        "output '%s' of derivation '%s' isn’t built", outputName,
+                        store.printStorePath(inputDrv));
+                inputRealisations.insert(*thisRealisation);
+            }
         }
-    }
+        if (!inputNode.value.empty()) {
+            auto d = makeConstantStorePathRef(inputDrv);
+            for (const auto & [outputName, childNode] : inputNode.childMap) {
+                SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
+                accumRealisations(
+                    // TODO deep resolutions for dynamic derivations, issue #8947, would go here.
+                    resolveDerivedPath(store, next),
+                    childNode);
+            }
+        }
+    };
+
+    for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
+        accumRealisations(inputDrv, inputNode);
 
     auto info = store.queryPathInfo(outputPath);
 

src/libstore/tests/derivation.cc

@@ -42,6 +42,26 @@ class ImpureDerivationTest : public DerivationTest
     }
 };
 
+TEST_F(DerivationTest, BadATerm_version) {
+    ASSERT_THROW(
+        parseDerivation(
+            *store,
+            R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+            "whatever",
+            mockXpSettings),
+        FormatError);
+}
+
+TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
+    ASSERT_THROW(
+        parseDerivation(
+            *store,
+            R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+            "dyn-dep-derivation",
+            mockXpSettings),
+        FormatError);
+}
+
 #define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
     TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {    \
         using nlohmann::literals::operator "" _json;           \
@@ -143,35 +163,37 @@ TEST_JSON(ImpureDerivationTest, impure,
 
 #undef TEST_JSON
 
-#define TEST_JSON(NAME, STR, VAL)                                \
+#define TEST_JSON(FIXTURE, NAME, STR, VAL)                       \
-    TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) {    \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) {           \
         using nlohmann::literals::operator "" _json;             \
         ASSERT_EQ(                                               \
             STR ## _json,                                        \
             (VAL).toJSON(*store));                               \
     }                                                            \
                                                                  \
-    TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) {  \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) {         \
         using nlohmann::literals::operator "" _json;             \
         ASSERT_EQ(                                               \
             (VAL),                                               \
             Derivation::fromJSON(                                \
                 *store,                                          \
-                STR ## _json));                                  \
+                STR ## _json,                                    \
+                mockXpSettings));                                \
     }
 
-#define TEST_ATERM(NAME, STR, VAL, DRV_NAME)                     \
+#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME)            \
-    TEST_F(DerivationTest, Derivation_ ## NAME ## _to_aterm) {   \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) {          \
         ASSERT_EQ(                                               \
             STR,                                                 \
             (VAL).unparse(*store, false));                       \
     }                                                            \
                                                                  \
-    TEST_F(DerivationTest, Derivation_ ## NAME ## _from_aterm) { \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) {        \
         auto parsed = parseDerivation(                           \
-             *store,                                             \
+            *store,                                              \
-             STR,                                                \
+            STR,                                                 \
-             DRV_NAME);                                          \
+            DRV_NAME,                                            \
+            mockXpSettings);                                     \
         ASSERT_EQ(                                               \
             (VAL).toJSON(*store),                                \
             parsed.toJSON(*store));                              \
@@ -187,11 +209,15 @@ Derivation makeSimpleDrv(const Store & store) {
         store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
     };
     drv.inputDrvs = {
-        {
+        .map = {
-            store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
             {
-                "cat",
+                store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
-                "dog",
+                {
+                    .value = {
+                        "cat",
+                        "dog",
+                    },
+                },
             },
         },
     };
@@ -210,17 +236,20 @@ Derivation makeSimpleDrv(const Store & store) {
     return drv;
 }
 
-TEST_JSON(simple,
+TEST_JSON(DerivationTest, simple,
     R"({
       "name": "simple-derivation",
       "inputSrcs": [
         "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
       ],
       "inputDrvs": {
-        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
-          "cat",
+          "dynamicOutputs": {},
-          "dog"
+          "outputs": [
-        ]
+            "cat",
+            "dog"
+          ]
+        }
       },
       "system": "wasm-sel4",
       "builder": "foo",
@@ -235,11 +264,105 @@ TEST_JSON(simple,
     })",
     makeSimpleDrv(*store))
 
-TEST_ATERM(simple,
+TEST_ATERM(DerivationTest, simple,
     R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
     makeSimpleDrv(*store),
     "simple-derivation")
 
+Derivation makeDynDepDerivation(const Store & store) {
+    Derivation drv;
+    drv.name = "dyn-dep-derivation";
+    drv.inputSrcs = {
+        store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
+    };
+    drv.inputDrvs = {
+        .map = {
+            {
+                store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
+                DerivedPathMap<StringSet>::ChildNode {
+                    .value = {
+                        "cat",
+                        "dog",
+                    },
+                    .childMap = {
+                        {
+                            "cat",
+                            DerivedPathMap<StringSet>::ChildNode {
+                                .value = {
+                                    "kitten",
+                                },
+                            },
+                        },
+                        {
+                            "goose",
+                            DerivedPathMap<StringSet>::ChildNode {
+                                .value = {
+                                    "gosling",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    };
+    drv.platform = "wasm-sel4";
+    drv.builder = "foo";
+    drv.args = {
+        "bar",
+        "baz",
+    };
+    drv.env = {
+        {
+            "BIG_BAD",
+            "WOLF",
+        },
+    };
+    return drv;
+}
+
+TEST_JSON(DynDerivationTest, dynDerivationDeps,
+    R"({
+      "name": "dyn-dep-derivation",
+      "inputSrcs": [
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
+      ],
+      "inputDrvs": {
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
+          "dynamicOutputs": {
+            "cat": {
+              "dynamicOutputs": {},
+              "outputs": ["kitten"]
+            },
+            "goose": {
+              "dynamicOutputs": {},
+              "outputs": ["gosling"]
+            }
+          },
+          "outputs": [
+            "cat",
+            "dog"
+          ]
+        }
+      },
+      "system": "wasm-sel4",
+      "builder": "foo",
+      "args": [
+        "bar",
+        "baz"
+      ],
+      "env": {
+        "BIG_BAD": "WOLF"
+      },
+      "outputs": {}
+    })",
+    makeDynDepDerivation(*store))
+
+TEST_ATERM(DynDerivationTest, dynDerivationDeps,
+    R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+    makeDynDepDerivation(*store),
+    "dyn-dep-derivation")
+
 #undef TEST_JSON
 #undef TEST_ATERM
 

src/libutil/comparator.hh

@@ -1,21 +1,48 @@
 #pragma once
 ///@file
 
+#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \
+    PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const;
+#define DECLARE_EQUAL(prefix, qualification, my_type) \
+    DECLARE_ONE_CMP(prefix, qualification, ==, my_type)
+#define DECLARE_LEQ(prefix, qualification, my_type) \
+    DECLARE_ONE_CMP(prefix, qualification, <, my_type)
+#define DECLARE_NEQ(prefix, qualification, my_type) \
+    DECLARE_ONE_CMP(prefix, qualification, !=, my_type)
+
+#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \
+    PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \
+      __VA_OPT__(const MY_TYPE * me = this;) \
+      auto fields1 = std::make_tuple( __VA_ARGS__ ); \
+      __VA_OPT__(me = &other;) \
+      auto fields2 = std::make_tuple( __VA_ARGS__ ); \
+      return fields1 COMPARATOR fields2; \
+    }
+#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \
+    GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args)
+#define GENERATE_LEQ(prefix, qualification, my_type, args...) \
+    GENERATE_ONE_CMP(prefix, qualification, <, my_type, args)
+#define GENERATE_NEQ(prefix, qualification, my_type, args...) \
+    GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args)
+
 /**
  * Declare comparison methods without defining them.
  */
-#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \
-    bool operator COMPARATOR(const MY_TYPE & other) const;
-#define DECLARE_EQUAL(my_type) \
-    DECLARE_ONE_CMP(==, my_type)
-#define DECLARE_LEQ(my_type) \
-    DECLARE_ONE_CMP(<, my_type)
-#define DECLARE_NEQ(my_type) \
-    DECLARE_ONE_CMP(!=, my_type)
 #define DECLARE_CMP(my_type) \
-    DECLARE_EQUAL(my_type) \
+    DECLARE_EQUAL(,,my_type) \
-    DECLARE_LEQ(my_type) \
+    DECLARE_LEQ(,,my_type) \
-    DECLARE_NEQ(my_type)
+    DECLARE_NEQ(,,my_type)
+
+/**
+ * @param prefix This is for something before each declaration like
+ * `template<classname Foo>`.
+ *
+ * @param my_type the type are defining operators for.
+ */
+#define DECLARE_CMP_EXT(prefix, qualification, my_type) \
+    DECLARE_EQUAL(prefix, qualification, my_type) \
+    DECLARE_LEQ(prefix, qualification, my_type) \
+    DECLARE_NEQ(prefix, qualification, my_type)
 
 /**
  * Awful hacky generation of the comparison operators by doing a lexicographic
@@ -33,18 +60,18 @@
  * }
  * ```
  */
-#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \
-    bool operator COMPARATOR(const MY_TYPE& other) const { \
-      __VA_OPT__(const MY_TYPE* me = this;) \
-      auto fields1 = std::make_tuple( __VA_ARGS__ ); \
-      __VA_OPT__(me = &other;) \
-      auto fields2 = std::make_tuple( __VA_ARGS__ ); \
-      return fields1 COMPARATOR fields2; \
-    }
-#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
-#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
-#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
 #define GENERATE_CMP(args...) \
-    GENERATE_EQUAL(args) \
+    GENERATE_EQUAL(,,args) \
-    GENERATE_LEQ(args) \
+    GENERATE_LEQ(,,args) \
-    GENERATE_NEQ(args)
+    GENERATE_NEQ(,,args)
+
+/**
+ * @param prefix This is for something before each declaration like
+ * `template<classname Foo>`.
+ *
+ * @param my_type the type are defining operators for.
+ */
+#define GENERATE_CMP_EXT(prefix, my_type, args...) \
+    GENERATE_EQUAL(prefix, my_type ::, my_type, args) \
+    GENERATE_LEQ(prefix, my_type ::, my_type, args) \
+    GENERATE_NEQ(prefix, my_type ::, my_type, args)

src/nix-build/nix-build.cc

@@ -406,8 +406,22 @@ static void main_nix_build(int argc, char * * argv)
             }
         }
 
+        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
+
+        accumDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            if (!inputNode.value.empty())
+                pathsToBuild.push_back(DerivedPath::Built {
+                    .drvPath = inputDrv,
+                    .outputs = OutputsSpec::Names { inputNode.value },
+                });
+            for (const auto & [outputName, childNode] : inputNode.childMap)
+                accumDerivedPath(
+                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                    childNode);
+        };
+
         // Build or fetch all dependencies of the derivation.
-        for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) {
+        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) {
             // To get around lambda capturing restrictions in the
             // standard.
             const auto & inputDrv = inputDrv0;
@@ -416,10 +430,7 @@ static void main_nix_build(int argc, char * * argv)
                         return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude));
                     }))
             {
-                pathsToBuild.push_back(DerivedPath::Built {
+                accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode);
-                    .drvPath = makeConstantStorePathRef(inputDrv),
-                    .outputs = OutputsSpec::Names { inputOutputs },
-                });
                 pathsToCopy.insert(inputDrv);
             }
         }
@@ -482,13 +493,21 @@ static void main_nix_build(int argc, char * * argv)
 
         if (env.count("__json")) {
             StorePathSet inputs;
-            for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) {
+
-                auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath);
+            std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
-                for (auto & i : wantedDepOutputs) {
+
+            accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+                auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv);
+                for (auto & i : inputNode.value) {
                     auto o = outputs.at(i);
                     store->computeFSClosure(*o, inputs);
                 }
-            }
+                for (const auto & [outputName, childNode] : inputNode.childMap)
+                    accumInputClosure(*outputs.at(outputName), childNode);
+            };
+
+            for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
+                accumInputClosure(inputDrv, inputNode);
 
             ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv);
 

src/nix/app.cc

@@ -20,15 +20,22 @@ StringPairs resolveRewrites(
     const std::vector<BuiltPathWithResult> & dependencies)
 {
     StringPairs res;
-    for (auto & dep : dependencies)
+    if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
+        for (auto & dep : dependencies) {
-            if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
+            if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path)) {
-                for (auto & [ outputName, outputPath ] : drvDep->outputs)
+                for (auto & [ outputName, outputPath ] : drvDep->outputs) {
                     res.emplace(
-                        DownstreamPlaceholder::unknownCaOutput(
+                        DownstreamPlaceholder::fromSingleDerivedPathBuilt(
-                            drvDep->drvPath->outPath(), outputName).render(),
+                            SingleDerivedPath::Built {
+                                .drvPath = make_ref<SingleDerivedPath>(drvDep->drvPath->discardOutputPath()),
+                                .output = outputName,
+                            }).render(),
                         store.printStorePath(outputPath)
                     );
+                }
+            }
+        }
+    }
     return res;
 }
 

tests/dyn-drv/dep-built-drv.sh

@@ -6,4 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
 
 clearStore
 
-expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported"
+out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link)
+
+diff -r $out1 $out2