From f7d9f7c3381acef38e4db2bb2f9e0287c289be54 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Mar 2021 05:48:01 +0000 Subject: [PATCH 01/10] Pull out Buildable into its own file/header in libnixstore --- src/libcmd/installables.cc | 25 ------------------------- src/libcmd/installables.hh | 22 +--------------------- src/libstore/buildable.cc | 33 +++++++++++++++++++++++++++++++++ src/libstore/buildable.hh | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 46 deletions(-) create mode 100644 src/libstore/buildable.cc create mode 100644 src/libstore/buildable.hh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 898e642a5..ca416b9ee 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -20,31 +20,6 @@ namespace nix { -nlohmann::json BuildableOpaque::toJSON(ref store) const { - nlohmann::json res; - res["path"] = store->printStorePath(path); - return res; -} - -nlohmann::json BuildableFromDrv::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = path ? store->printStorePath(*path) : ""; - } - return res; -} - -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { - auto res = nlohmann::json::array(); - for (const Buildable & buildable : buildables) { - std::visit([&res, store](const auto & buildable) { - res.push_back(buildable.toJSON(store)); - }, buildable); - } - return res; -} - void completeFlakeInputPath( ref evalState, const FlakeRef & flakeRef, diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b714f097b..d31afd3d5 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,13 +2,12 @@ #include "util.hh" #include "path.hh" +#include "buildable.hh" #include "eval.hh" #include "flake/flake.hh" #include -#include - namespace nix { struct DrvInfo; @@ -16,25 +15,6 @@ struct SourceExprCommand; namespace eval_cache { class EvalCache; class AttrCursor; } -struct BuildableOpaque { - StorePath path; - nlohmann::json toJSON(ref store) const; -}; - -struct BuildableFromDrv { - StorePath drvPath; - std::map> outputs; - nlohmann::json toJSON(ref store) const; -}; - -typedef std::variant< - BuildableOpaque, - BuildableFromDrv -> Buildable; - -typedef std::vector Buildables; -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); - struct App { std::vector context; diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc new file mode 100644 index 000000000..5cba45b1d --- /dev/null +++ b/src/libstore/buildable.cc @@ -0,0 +1,33 @@ +#include "buildable.hh" +#include "store-api.hh" + +#include + +namespace nix { + +nlohmann::json BuildableOpaque::toJSON(ref store) const { + nlohmann::json res; + res["path"] = store->printStorePath(path); + return res; +} + +nlohmann::json BuildableFromDrv::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = path ? store->printStorePath(*path) : ""; + } + return res; +} + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { + auto res = nlohmann::json::array(); + for (const Buildable & buildable : buildables) { + std::visit([&res, store](const auto & buildable) { + res.push_back(buildable.toJSON(store)); + }, buildable); + } + return res; +} + +} diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh new file mode 100644 index 000000000..6177237be --- /dev/null +++ b/src/libstore/buildable.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "util.hh" +#include "path.hh" + +#include + +#include + +namespace nix { + +class Store; + +struct BuildableOpaque { + StorePath path; + nlohmann::json toJSON(ref store) const; +}; + +struct BuildableFromDrv { + StorePath drvPath; + std::map> outputs; + nlohmann::json toJSON(ref store) const; +}; + +typedef std::variant< + BuildableOpaque, + BuildableFromDrv +> Buildable; + +typedef std::vector Buildables; + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); + +} From 7a2b566dc8f0f94fdd6acbce90e47cd967f9f134 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 00:47:00 +0000 Subject: [PATCH 02/10] Move `StorePathWithOutputs` into its own header/file In the following commits it will become less prevalent. --- src/libcmd/installables.hh | 1 + src/libstore/derivations.cc | 8 ------- src/libstore/path-with-outputs.cc | 36 +++++++++++++++++++++++++++++++ src/libstore/path-with-outputs.hh | 17 +++++++++++++++ src/libstore/path.cc | 15 ------------- src/libstore/path.hh | 10 --------- src/libstore/store-api.cc | 7 ------ src/libstore/store-api.hh | 1 + 8 files changed, 55 insertions(+), 40 deletions(-) create mode 100644 src/libstore/path-with-outputs.cc create mode 100644 src/libstore/path-with-outputs.hh diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index d31afd3d5..e5c6fe208 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "path-with-outputs.hh" #include "buildable.hh" #include "eval.hh" #include "flake/flake.hh" diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fe98182bb..f6defd98f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -590,14 +590,6 @@ std::map staticOutputHashes(Store& store, const Derivation& d } -std::string StorePathWithOutputs::to_string(const Store & store) const -{ - return outputs.empty() - ? store.printStorePath(path) - : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); -} - - bool wantOutput(const string & output, const std::set & wanted) { return wanted.empty() || wanted.find(output) != wanted.end(); diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc new file mode 100644 index 000000000..ba15df0a9 --- /dev/null +++ b/src/libstore/path-with-outputs.cc @@ -0,0 +1,36 @@ +#include "store-api.hh" + +namespace nix { + +std::string StorePathWithOutputs::to_string(const Store & store) const +{ + return outputs.empty() + ? store.printStorePath(path) + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); +} + + +std::pair parsePathWithOutputs(std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? std::make_pair(s, std::set()) + : std::make_pair(((std::string_view) s).substr(0, n), + tokenizeString>(((std::string_view) s).substr(n + 1), ",")); +} + + +StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) +{ + auto [path, outputs] = nix::parsePathWithOutputs(s); + return {parseStorePath(path), std::move(outputs)}; +} + + +StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const +{ + auto [path2, outputs] = nix::parsePathWithOutputs(path); + return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; +} + +} diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh new file mode 100644 index 000000000..a9e3fc7c2 --- /dev/null +++ b/src/libstore/path-with-outputs.hh @@ -0,0 +1,17 @@ +#pragma once + +#include "path.hh" + +namespace nix { + +struct StorePathWithOutputs +{ + StorePath path; + std::set outputs; + + std::string to_string(const Store & store) const; +}; + +std::pair parsePathWithOutputs(std::string_view s); + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index dc9dc3897..e642abcd5 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -std::pair parsePathWithOutputs(std::string_view s) -{ - size_t n = s.find("!"); - return n == s.npos - ? std::make_pair(s, std::set()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString>(((std::string_view) s).substr(n + 1), ",")); -} - -StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) -{ - auto [path, outputs] = nix::parsePathWithOutputs(s); - return {parseStorePath(path), std::move(outputs)}; -} - } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..06ba0663b 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -69,16 +69,6 @@ typedef std::map> StorePathCAMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -struct StorePathWithOutputs -{ - StorePath path; - std::set outputs; - - std::string to_string(const Store & store) const; -}; - -std::pair parsePathWithOutputs(std::string_view s); - } namespace std { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5e321cedf..e3500872c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -53,13 +53,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const } -StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const -{ - auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; -} - - /* Store paths have the following form: = /- diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5d19e8949..7adbe3b17 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,6 +2,7 @@ #include "realisation.hh" #include "path.hh" +#include "path-with-outputs.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" From 32f4454b9fa3ac30d58e738ece322eb19a0728ba Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 01:06:08 +0000 Subject: [PATCH 03/10] Move `StorePathWithOutput` utilities out of store class These are by no means part of the notion of a store, but rather are things that happen to use stores. (Or put another way, there's no way we'd make them virtual methods any time soon.) It's better to move them out of that too-big class then. Also, this helps us remove StorePathWithOutputs from the Store interface altogether next commit. --- src/libexpr/get-drvs.cc | 2 +- src/libstore/daemon.cc | 4 ++-- src/libstore/path-with-outputs.cc | 12 ++++++------ src/libstore/path-with-outputs.hh | 9 +++++++++ src/libstore/store-api.hh | 7 ------- src/nix-store/nix-store.cc | 4 ++-- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1a3990ea1..7793f26ff 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -19,7 +19,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { - auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); + auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); this->drvPath = store->printStorePath(drvPath); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f28ab6438..48706bff8 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildPaths: { std::vector drvs; for (auto & s : readStrings(from)) - drvs.push_back(store->parsePathWithOutputs(s)); + drvs.push_back(parsePathWithOutputs(*store, s)); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -861,7 +861,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryMissing: { std::vector targets; for (auto & s : readStrings(from)) - targets.push_back(store->parsePathWithOutputs(s)); + targets.push_back(parsePathWithOutputs(*store, s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index ba15df0a9..a898ad09c 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -20,17 +20,17 @@ std::pair parsePathWithOutputs(std::string_view s) } -StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) { - auto [path, outputs] = nix::parsePathWithOutputs(s); - return {parseStorePath(path), std::move(outputs)}; + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; } -StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs) { - auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index a9e3fc7c2..0e34b5aa1 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -14,4 +14,13 @@ struct StorePathWithOutputs std::pair parsePathWithOutputs(std::string_view s); +class Store; + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path + and the outputs. */ +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7adbe3b17..da7ac4460 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -262,11 +262,6 @@ public: PathSet printStorePathSet(const StorePathSet & path) const; - /* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ - StorePathWithOutputs parsePathWithOutputs(const string & s); - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ std::string showPaths(const StorePathSet & paths); @@ -290,8 +285,6 @@ public: result. */ StorePath followLinksToStorePath(std::string_view path) const; - StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; - /* Constructs a unique store path name. */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b684feccb..bfd1299fc 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -128,7 +128,7 @@ static void opRealise(Strings opFlags, Strings opArgs) std::vector paths; for (auto & i : opArgs) - paths.push_back(store->followLinksToStorePathWithOutputs(i)); + paths.push_back(followLinksToStorePathWithOutputs(*store, i)); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; @@ -873,7 +873,7 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector paths; for (auto & s : readStrings(in)) - paths.push_back(store->parsePathWithOutputs(s)); + paths.push_back(parsePathWithOutputs(*store, s)); getBuildSettings(); From 255d145ba7ac907d1cba8d088da556b591627756 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 03:50:41 +0000 Subject: [PATCH 04/10] Use `BuildableReq` for `buildPaths` and `ensurePath` This avoids an ambiguity where the `StorePathWithOutputs { drvPath, {} }` could mean "build `brvPath`" or "substitute `drvPath`" depending on context. It also brings the internals closer in line to the new CLI, by generalizing the `Buildable` type is used there and makes that distinction already. In doing so, relegate `StorePathWithOutputs` to being a type just for backwards compatibility (CLI and RPC). --- src/libcmd/installables.cc | 7 +-- src/libexpr/get-drvs.cc | 1 + src/libexpr/primops.cc | 12 +++-- src/libmain/shared.cc | 2 +- src/libmain/shared.hh | 3 +- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/build/entry-points.cc | 16 +++--- src/libstore/build/local-derivation-goal.cc | 52 +++++++++++++------ src/libstore/build/local-derivation-goal.hh | 1 + src/libstore/build/worker.cc | 6 +-- src/libstore/buildable.cc | 47 +++++++++++++++++ src/libstore/buildable.hh | 29 +++++++++-- src/libstore/daemon.cc | 21 +++++--- src/libstore/legacy-ssh-store.cc | 16 ++++-- src/libstore/misc.cc | 49 +++++++++--------- src/libstore/path-with-outputs.cc | 35 +++++++++++++ src/libstore/path-with-outputs.hh | 9 ++++ src/libstore/remote-store.cc | 57 +++++++++++++++++---- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 8 +-- src/libstore/store-api.hh | 6 +-- src/libstore/worker-protocol.hh | 22 ++++++++ src/nix-build/nix-build.cc | 4 +- src/nix-env/nix-env.cc | 30 ++++++----- src/nix-env/user-env.cc | 9 +++- src/nix-store/nix-store.cc | 11 ++-- src/nix/bundle.cc | 4 +- src/nix/develop.cc | 3 +- src/nix/flake.cc | 5 +- src/nix/profile.cc | 15 +++--- src/nix/run.cc | 2 +- 31 files changed, 364 insertions(+), 126 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index ca416b9ee..b68c5f6a7 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -679,19 +679,20 @@ Buildables build(ref store, Realise mode, Buildables buildables; - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & i : installables) { for (auto & b : i->toBuildables()) { std::visit(overloaded { [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path}); + pathsToBuild.push_back(bo); }, [&](BuildableFromDrv bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); - pathsToBuild.push_back({bfd.drvPath, outputNames}); + pathsToBuild.push_back( + BuildableReqFromDrv{bfd.drvPath, outputNames}); }, }, b); buildables.push_back(std::move(b)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7793f26ff..f774e6493 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "eval-inline.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include #include diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d1afa768..24bc34b74 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { - std::vector drvs; + std::vector drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -43,7 +43,7 @@ void EvalState::realiseContext(const PathSet & context) if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); + drvs.push_back({ctx, {outputName}}); } } @@ -51,14 +51,16 @@ void EvalState::realiseContext(const PathSet & context) if (!evalSettings.enableImportFromDerivation) throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", - store->printStorePath(drvs.begin()->path)); + store->printStorePath(drvs.begin()->drvPath)); /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + std::vector buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d }); + store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); + store->buildPaths(buildReqs); /* Add the output of this derivations to the allowed paths. */ diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5baaff3e9..20027e099 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() } -void printMissing(ref store, const std::vector & paths, Verbosity lvl) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index edc7b5efa..18e0fb57d 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -4,6 +4,7 @@ #include "args.hh" #include "common-args.hh" #include "path.hh" +#include "buildable.hh" #include @@ -42,7 +43,7 @@ struct StorePathWithOutputs; void printMissing( ref store, - const std::vector & paths, + const std::vector & paths, Verbosity lvl = lvlInfo); void printMissing(ref store, const StorePathSet & willBuild, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e7be517e..8680d0bce 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); + to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs })); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); + to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() })); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 686364440..d1973d78b 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,16 +6,20 @@ namespace nix { -void Store::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) { Worker worker(*this); Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); + for (auto & br : reqs) { + std::visit(overloaded { + [&](BuildableReqFromDrv bfd) { + goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); + }, + [&](BuildableOpaque bo) { + goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); + }, + }, br); } worker.run(goals); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8ef43c225..c245527c9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1190,6 +1190,26 @@ void LocalDerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } + +static StorePath pathPartOfReq(const BuildableReq & req) +{ + return std::visit(overloaded { + [&](BuildableOpaque bo) { + return bo.path; + }, + [&](BuildableReqFromDrv bfd) { + return bfd.drvPath; + }, + }, req); +} + + +bool LocalDerivationGoal::isAllowed(const BuildableReq & req) +{ + return this->isAllowed(pathPartOfReq(req)); +} + + struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; @@ -1312,25 +1332,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // an allowed derivation { throw Error("queryRealisation"); } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + for (auto & req : paths) { + if (!goal.isAllowed(req)) + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req)); } next->buildPaths(paths, buildMode); for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); + auto p = std::get_if(&path); + if (!p) continue; + auto & bfd = *p; + auto outputs = next->queryDerivationOutputMap(bfd.drvPath); + for (auto & [outputName, outputPath] : outputs) + if (wantOutput(outputName, bfd.outputs)) + newPaths.insert(outputPath); } StorePathSet closure; @@ -1358,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override { @@ -1366,12 +1388,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo client about what paths will be built/substituted or are already present. Probably not a big deal. */ - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); + std::vector allowed; + for (auto & req : targets) { + if (goal.isAllowed(req)) + allowed.emplace_back(req); else - unknown.insert(path.path); + unknown.insert(pathPartOfReq(req)); } next->queryMissing(allowed, willBuild, willSubstitute, diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 47b818a8b..edb93f84e 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -116,6 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } + bool isAllowed(const BuildableReq & req); friend struct RestrictedStore; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 616b17e61..fef4cb0cb 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; + std::vector topPaths; for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->storePath}); + topPaths.push_back(BuildableOpaque{goal->storePath}); } } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 5cba45b1d..63ca1779e 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,6 +11,7 @@ nlohmann::json BuildableOpaque::toJSON(ref store) const { return res; } +template<> nlohmann::json BuildableFromDrv::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -30,4 +31,50 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) return res; } + +std::string BuildableOpaque::to_string(const Store & store) const { + return store.printStorePath(path); +} + +template<> +std::string BuildableReqFromDrv::to_string(const Store & store) const { + return store.printStorePath(drvPath) + + "!" + + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); +} + +std::string to_string(const Store & store, const BuildableReq & req) +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + req); +} + + +BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) +{ + return {store.parseStorePath(s)}; +} + +template<> +BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + assert(n != s.npos); + auto drvPath = store.parseStorePath(s.substr(0, n)); + auto outputsS = s.substr(n + 1); + std::set outputs; + if (outputsS != "*") + outputs = tokenizeString>(outputsS); + return {drvPath, outputs}; +} + +BuildableReq parseBuildableReq(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? (BuildableReq) BuildableOpaque::parse(store, s) + : (BuildableReq) BuildableReqFromDrv::parse(store, s); +} + } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 6177237be..db78316bd 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "path.hh" #include @@ -13,19 +14,37 @@ class Store; struct BuildableOpaque { StorePath path; + nlohmann::json toJSON(ref store) const; + std::string to_string(const Store & store) const; + static BuildableOpaque parse(const Store & store, std::string_view); }; -struct BuildableFromDrv { +template +struct BuildableForFromDrv { StorePath drvPath; - std::map> outputs; + Outputs outputs; + nlohmann::json toJSON(ref store) const; + std::string to_string(const Store & store) const; + static BuildableForFromDrv parse(const Store & store, std::string_view); }; -typedef std::variant< +template +using BuildableFor = std::variant< BuildableOpaque, - BuildableFromDrv -> Buildable; + BuildableForFromDrv +>; + +typedef BuildableForFromDrv> BuildableReqFromDrv; +typedef BuildableFor> BuildableReq; + +std::string to_string(const Store & store, const BuildableReq &); + +BuildableReq parseBuildableReq(const Store & store, std::string_view); + +typedef BuildableForFromDrv>> BuildableFromDrv; +typedef BuildableFor>> Buildable; typedef std::vector Buildables; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 48706bff8..6b527dcb2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,6 +2,7 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" @@ -259,6 +260,18 @@ static void writeValidPathInfo( } } +static std::vector readBuildableReqs(Store & store, unsigned int clientVersion, Source & from) +{ + std::vector reqs; + if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + reqs = worker_proto::read(store, from, Phantom> {}); + } else { + for (auto & s : readStrings(from)) + reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq()); + } + return reqs; +} + static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -493,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopBuildPaths: { - std::vector drvs; - for (auto & s : readStrings(from)) - drvs.push_back(parsePathWithOutputs(*store, s)); + auto drvs = readBuildableReqs(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -859,9 +870,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryMissing: { - std::vector targets; - for (auto & s : readStrings(from)) - targets.push_back(parsePathWithOutputs(*store, s)); + auto targets = readBuildableReqs(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a9f53bad9..1cb977be6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,6 +3,7 @@ #include "remote-store.hh" #include "serve-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" @@ -266,14 +267,23 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildPaths; Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(*this)); + }, + [&](StorePath drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + }, sOrDrvPath); + } conn->to << ss; putBuildSettings(*conn); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f58816ad8..e702a4f9e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -117,7 +117,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) return std::nullopt; } -void Store::queryMissing(const std::vector & targets, +void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, uint64_t & downloadSize_, uint64_t & narSize_) { @@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector & targets, Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { @@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); + pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second })); }; auto checkOutput = [&]( @@ -177,24 +177,25 @@ void Store::queryMissing(const std::vector & targets, drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); + pool.enqueue(std::bind(doPath, BuildableOpaque { path } )); } } } }; - doPath = [&](const StorePathWithOutputs & path) { + doPath = [&](const BuildableReq & req) { { auto state(state_.lock()); - if (!state->done.insert(path.to_string(*this)).second) return; + if (!state->done.insert(to_string(*this, req)).second) return; } - if (path.path.isDerivation()) { - if (!isValidPath(path.path)) { + std::visit(overloaded { + [&](BuildableReqFromDrv bfd) { + if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bfd.drvPath); return; } @@ -202,52 +203,54 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; } - if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) + if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(StorePath(path.path), *drv); + auto drv = make_ref(derivationFromPath(bfd.drvPath)); + ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); } else - mustBuildDrv(path.path, *drv); + mustBuildDrv(bfd.drvPath, *drv); - } else { + }, + [&](BuildableOpaque bo) { - if (isValidPath(path.path)) return; + if (isValidPath(bo.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); + querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bo.path); return; } - auto info = infos.find(path.path); + auto info = infos.find(bo.path); assert(info != infos.end()); { auto state(state_.lock()); - state->willSubstitute.insert(path.path); + state->willSubstitute.insert(bo.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); - } + pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); + }, + }, req); }; for (auto & path : targets) diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index a898ad09c..353286ac6 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,3 +1,4 @@ +#include "path-with-outputs.hh" #include "store-api.hh" namespace nix { @@ -10,6 +11,40 @@ std::string StorePathWithOutputs::to_string(const Store & store) const } +BuildableReq StorePathWithOutputs::toBuildableReq() const +{ + if (!outputs.empty() || path.isDerivation()) + return BuildableReqFromDrv { path, outputs }; + else + return BuildableOpaque { path }; +} + + +std::vector toBuildableReqs(const std::vector ss) +{ + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toBuildableReq()); + return reqs; +} + + +std::variant StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p) +{ + return std::visit(overloaded { + [&](BuildableOpaque bo) -> std::variant { + if (bo.path.isDerivation()) { + // drv path gets interpreted as "build", not "get drv file itself" + return bo.path; + } + return StorePathWithOutputs { bo.path }; + }, + [&](BuildableReqFromDrv bfd) -> std::variant { + return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + }, + }, p); +} + + std::pair parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 0e34b5aa1..870cac08e 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,6 +1,9 @@ #pragma once +#include + #include "path.hh" +#include "buildable.hh" namespace nix { @@ -10,8 +13,14 @@ struct StorePathWithOutputs std::set outputs; std::string to_string(const Store & store) const; + + BuildableReq toBuildableReq() const; + + static std::variant tryFromBuildableReq(const BuildableReq &); }; +std::vector toBuildableReqs(const std::vector); + std::pair parsePathWithOutputs(std::string_view s); class Store; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ccf095dc2..de1c95ed6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "util.hh" +#include "path-with-outputs.hh" #include "remote-fs-accessor.hh" #include "remote-store.hh" #include "worker-protocol.hh" @@ -50,6 +51,19 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) out << renderContentAddress(ca); } + +BuildableReq read(const Store & store, Source & from, Phantom _) +{ + auto s = readString(from); + return parseBuildableReq(store, s); +} + +void write(const Store & store, Sink & out, const BuildableReq & req) +{ + out << to_string(store, req); +} + + Realisation read(const Store & store, Source & from, Phantom _) { std::string rawInput = readString(from); @@ -58,8 +72,12 @@ Realisation read(const Store & store, Source & from, Phantom _) "remote-protocol" ); } + void write(const Store & store, Sink & out, const Realisation & realisation) -{ out << realisation.toJSON().dump(); } +{ + out << realisation.toJSON().dump(); +} + DrvOutput read(const Store & store, Source & from, Phantom _) { @@ -652,16 +670,36 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & return {Realisation{.id = id, .outPath = *outPaths.begin()}}; } +static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +{ + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + worker_proto::write(store, conn->to, reqs); + } else { + Strings ss; + for (auto & p : reqs) { + auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(store)); + }, + [&](StorePath drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn->daemonVersion), + GET_PROTOCOL_MINOR(conn->daemonVersion)); + }, + }, sOrDrvPath); + } + conn->to << ss; + } +} -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeBuildableReqs(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -800,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s } -void RemoteStore::queryMissing(const std::vector & targets, +void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { @@ -811,10 +849,7 @@ void RemoteStore::queryMissing(const std::vector & targets // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing; - Strings ss; - for (auto & p : targets) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeBuildableReqs(*this, conn, targets); conn.processStderr(); willBuild = worker_proto::read(*this, conn->from, Phantom {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b3a9910a3..20d366038 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -108,7 +108,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e3500872c..8b60bdc62 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath, void Store::substitutePaths(const StorePathSet & paths) { - std::vector paths2; + std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back({path}); + paths2.push_back(BuildableOpaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back({p}); + std::vector subs; + for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index da7ac4460..59d0983df 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,7 +2,7 @@ #include "realisation.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "buildable.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" @@ -494,7 +494,7 @@ public: recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ virtual void buildPaths( - const std::vector & paths, + const std::vector & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an @@ -656,7 +656,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const std::vector & targets, + virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index be071dd78..0255726ac 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,9 +86,11 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); +MAKE_WORKER_PROTO(, BuildableReq); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); +MAKE_WORKER_PROTO(template, std::vector); MAKE_WORKER_PROTO(template, std::set); #define X_ template @@ -113,6 +115,26 @@ MAKE_WORKER_PROTO(X_, Y_); MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(, std::optional); +template +std::vector read(const Store & store, Source & from, Phantom> _) +{ + std::vector resSet; + auto size = readNum(from); + while (size--) { + resSet.push_back(read(store, from, Phantom {})); + } + return resSet; +} + +template +void write(const Store & store, Sink & out, const std::vector & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); + } +} + template std::set read(const Store & store, Source & from, Phantom> _) { diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 65b85b304..6f8a61261 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -12,6 +12,7 @@ #include "affinity.hh" #include "util.hh" #include "shared.hh" +#include "path-with-outputs.hh" #include "eval.hh" #include "eval-inline.hh" #include "get-drvs.hh" @@ -321,7 +322,8 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); - auto buildPaths = [&](const std::vector & paths) { + auto buildPaths = [&](const std::vector & paths0) { + auto paths = toBuildableReqs(paths0); /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0f10a4cbb..af1c69b87 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -6,6 +6,7 @@ #include "globals.hh" #include "names.hh" #include "profiles.hh" +#include "path-with-outputs.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" @@ -418,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - std::vector targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.push_back({state.store->parseStorePath(drvPath)}); + targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)}); else - targets.push_back({state.store->parseStorePath(i.queryOutPath())}); + targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -693,17 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); - if (drv.queryDrvPath() != "") { - std::vector paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}}; - printMissing(globals.state->store, paths); - if (globals.dryRun) return; - globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - } else { - printMissing(globals.state->store, - {{globals.state->store->parseStorePath(drv.queryOutPath())}}); - if (globals.dryRun) return; - globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); - } + std::vector paths { + (drv.queryDrvPath() != "") + ? (BuildableReq) (BuildableReqFromDrv { + globals.state->store->parseStorePath(drv.queryDrvPath()) + }) + : (BuildableReq) (BuildableOpaque { + globals.state->store->parseStorePath(drv.queryOutPath()) + }), + }; + printMissing(globals.state->store, paths); + if (globals.dryRun) return; + globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); debug(format("switching to new user environment")); Path generation = createGeneration( diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 168ac492b..0ccf960fb 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "derivations.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "local-fs-store.hh" #include "globals.hh" #include "shared.hh" @@ -41,7 +42,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); debug(format("building user environment dependencies")); - state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toBuildableReqs(drvsToBuild), + state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ StorePathSet references; @@ -136,7 +139,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug("building user environment"); std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); - state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toBuildableReqs(topLevelDrvs), + state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ auto store2 = state.store.dynamic_pointer_cast(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index bfd1299fc..21c1e547b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -10,6 +10,7 @@ #include "worker-protocol.hh" #include "graphml.hh" #include "legacy.hh" +#include "path-with-outputs.hh" #include #include @@ -62,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path}); + if (build) store->buildPaths({path.toBuildableReq()}); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -132,7 +133,9 @@ static void opRealise(Strings opFlags, Strings opArgs) uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->queryMissing( + toBuildableReqs(paths), + willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { std::vector paths2; @@ -148,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(paths, buildMode); + store->buildPaths(toBuildableReqs(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -879,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs) try { MonitorFdHup monitor(in.fd); - store->buildPaths(paths); + store->buildPaths(toBuildableReqs(paths)); out << 0; } catch (Error & e) { assert(e.status); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 48f4eb6e3..e86fbb3f7 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand auto evalState = getEvalState(); auto app = installable->toApp(*evalState); - store->buildPaths(app.context); + store->buildPaths(toBuildableReqs(app.context)); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; @@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); - store->buildPaths({{drvPath}}); + store->buildPaths({ BuildableReqFromDrv { drvPath } }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d0b140570..616e2073e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -3,6 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "derivations.hh" #include "affinity.hh" #include "progress-bar.hh" @@ -159,7 +160,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) auto shellDrvPath = writeDerivation(*store, drv); /* Build the derivation. */ - store->buildPaths({{shellDrvPath}}); + store->buildPaths({BuildableReqFromDrv{shellDrvPath}}); for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { auto & [_1, optPath] = outputAndOptPath; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a2b6c0303..9d6d22a43 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" +#include "path-with-outputs.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" @@ -292,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::vector drvPaths; + std::vector drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { @@ -461,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back({drvPath}); + drvPaths.push_back(BuildableReqFromDrv{drvPath}); } } } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 4d275f577..b96e71844 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast(installable)) { @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}}); + pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { @@ -260,12 +260,15 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile std::visit(overloaded { [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path, {}}); + pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, [&](BuildableFromDrv bfd) { + // TODO: Why are we querying if we know the output + // names already? Is it just to figure out what the + // default one is? for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back({bfd.drvPath, {output.first}}); + pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}}); element.storePaths.insert(output.second); } }, @@ -388,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - std::vector pathsToBuild; + std::vector pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); @@ -423,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME } } diff --git a/src/nix/run.cc b/src/nix/run.cc index ec9388234..2e9bb41cc 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon auto app = installable->toApp(*state); - state->store->buildPaths(app.context); + state->store->buildPaths(toBuildableReqs(app.context)); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); From 4fe41c6db390c0295d20f6365ebedaec8ec79e1d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:15:25 -0400 Subject: [PATCH 05/10] No templates for `Buildable` and `BuildableReq` --- src/libstore/buildable.cc | 3 --- src/libstore/buildable.hh | 30 +++++++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 63ca1779e..7892b94e4 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,7 +11,6 @@ nlohmann::json BuildableOpaque::toJSON(ref store) const { return res; } -template<> nlohmann::json BuildableFromDrv::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -36,7 +35,6 @@ std::string BuildableOpaque::to_string(const Store & store) const { return store.printStorePath(path); } -template<> std::string BuildableReqFromDrv::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" @@ -56,7 +54,6 @@ BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) return {store.parseStorePath(s)}; } -template<> BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index db78316bd..54e627271 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -20,31 +20,35 @@ struct BuildableOpaque { static BuildableOpaque parse(const Store & store, std::string_view); }; -template -struct BuildableForFromDrv { +struct BuildableReqFromDrv { StorePath drvPath; - Outputs outputs; + std::set outputs; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; - static BuildableForFromDrv parse(const Store & store, std::string_view); + static BuildableReqFromDrv parse(const Store & store, std::string_view); }; -template -using BuildableFor = std::variant< +using BuildableReq = std::variant< BuildableOpaque, - BuildableForFromDrv + BuildableReqFromDrv >; -typedef BuildableForFromDrv> BuildableReqFromDrv; -typedef BuildableFor> BuildableReq; - std::string to_string(const Store & store, const BuildableReq &); BuildableReq parseBuildableReq(const Store & store, std::string_view); -typedef BuildableForFromDrv>> BuildableFromDrv; -typedef BuildableFor>> Buildable; +struct BuildableFromDrv { + StorePath drvPath; + std::map> outputs; + + nlohmann::json toJSON(ref store) const; + static BuildableFromDrv parse(const Store & store, std::string_view); +}; + +using Buildable = std::variant< + BuildableOpaque, + BuildableFromDrv +>; typedef std::vector Buildables; From 9dfb97c987d8b9d6a3d15f016e40f22f91deb764 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:24:42 -0400 Subject: [PATCH 06/10] "newtype" BuildableReq This makes for better types errors and allows us to give it methods. --- src/libstore/build/derivation-goal.cc | 4 ++-- src/libstore/build/entry-points.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 4 ++-- src/libstore/buildable.cc | 6 +++--- src/libstore/buildable.hh | 14 +++++++++++--- src/libstore/misc.cc | 4 ++-- src/libstore/path-with-outputs.cc | 2 +- src/libstore/remote-store.cc | 4 ++-- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8680d0bce..8396abbcd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs })); + BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() })); + BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index d1973d78b..fc6294545 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -19,7 +19,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMo [&](BuildableOpaque bo) { goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); }, - }, br); + }, br.raw()); } worker.run(goals); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c245527c9..6cc384719 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1200,7 +1200,7 @@ static StorePath pathPartOfReq(const BuildableReq & req) [&](BuildableReqFromDrv bfd) { return bfd.drvPath; }, - }, req); + }, req.raw()); } @@ -1340,7 +1340,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo for (auto & req : paths) { if (!goal.isAllowed(req)) - throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req)); + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); } next->buildPaths(paths, buildMode); diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 7892b94e4..31fef2faa 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -41,11 +41,11 @@ std::string BuildableReqFromDrv::to_string(const Store & store) const { + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); } -std::string to_string(const Store & store, const BuildableReq & req) +std::string BuildableReq::to_string(const Store & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, - req); + this->raw()); } @@ -66,7 +66,7 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_ return {drvPath, outputs}; } -BuildableReq parseBuildableReq(const Store & store, std::string_view s) +BuildableReq BuildableReq::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); return n == s.npos diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 54e627271..8317f3995 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -28,14 +28,22 @@ struct BuildableReqFromDrv { static BuildableReqFromDrv parse(const Store & store, std::string_view); }; -using BuildableReq = std::variant< +using _BuildableReqRaw = std::variant< BuildableOpaque, BuildableReqFromDrv >; -std::string to_string(const Store & store, const BuildableReq &); +struct BuildableReq : _BuildableReqRaw { + using Raw = _BuildableReqRaw; + using Raw::Raw; -BuildableReq parseBuildableReq(const Store & store, std::string_view); + inline const Raw & raw() const { + return static_cast(*this); + } + + std::string to_string(const Store & store) const; + static BuildableReq parse(const Store & store, std::string_view); +}; struct BuildableFromDrv { StorePath drvPath; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e702a4f9e..abfae1502 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -187,7 +187,7 @@ void Store::queryMissing(const std::vector & targets, { auto state(state_.lock()); - if (!state->done.insert(to_string(*this, req)).second) return; + if (!state->done.insert(req.to_string(*this)).second) return; } std::visit(overloaded { @@ -250,7 +250,7 @@ void Store::queryMissing(const std::vector & targets, for (auto & ref : info->second.references) pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); }, - }, req); + }, req.raw()); }; for (auto & path : targets) diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 353286ac6..2898b8d4f 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -41,7 +41,7 @@ std::variant StorePathWithOutputs::tryFromBuild [&](BuildableReqFromDrv bfd) -> std::variant { return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; }, - }, p); + }, p.raw()); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index de1c95ed6..cb6402213 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -55,12 +55,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) BuildableReq read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return parseBuildableReq(store, s); + return BuildableReq::parse(store, s); } void write(const Store & store, Sink & out, const BuildableReq & req) { - out << to_string(store, req); + out << req.to_string(store); } From 9b805d36ac70545fc4c0d863e21e0c2e5f2518a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:48:18 -0400 Subject: [PATCH 07/10] Rename Buildable --- src/libcmd/command.cc | 6 +-- src/libcmd/command.hh | 4 +- src/libcmd/installables.cc | 44 ++++++++++----------- src/libcmd/installables.hh | 6 +-- src/libexpr/primops.cc | 6 +-- src/libmain/shared.cc | 2 +- src/libmain/shared.hh | 2 +- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/build/entry-points.cc | 6 +-- src/libstore/build/local-derivation-goal.cc | 16 ++++---- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/build/worker.cc | 6 +-- src/libstore/buildable.cc | 24 +++++------ src/libstore/buildable.hh | 37 +++++++++-------- src/libstore/daemon.cc | 12 +++--- src/libstore/legacy-ssh-store.cc | 4 +- src/libstore/misc.cc | 16 ++++---- src/libstore/path-with-outputs.cc | 18 ++++----- src/libstore/path-with-outputs.hh | 6 +-- src/libstore/remote-store.cc | 18 ++++----- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 8 ++-- src/libstore/store-api.hh | 4 +- src/libstore/worker-protocol.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 12 +++--- src/nix-env/user-env.cc | 4 +- src/nix-store/nix-store.cc | 8 ++-- src/nix/build.cc | 6 +-- src/nix/bundle.cc | 4 +- src/nix/develop.cc | 8 ++-- src/nix/flake.cc | 4 +- src/nix/log.cc | 6 +-- src/nix/profile.cc | 14 +++---- src/nix/run.cc | 2 +- 35 files changed, 165 insertions(+), 162 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index d29954f67..dc1fbc43f 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -162,7 +162,7 @@ void MixProfile::updateProfile(const StorePath & storePath) profile2, storePath)); } -void MixProfile::updateProfile(const Buildables & buildables) +void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) { if (!profile) return; @@ -170,10 +170,10 @@ void MixProfile::updateProfile(const Buildables & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { result.push_back(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index e66c697eb..9e18c6e51 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -216,7 +216,7 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -Buildables build(ref store, Realise mode, +DerivedPathsWithHints build(ref store, Realise mode, std::vector> installables, BuildMode bMode = bmNormal); std::set toStorePaths(ref store, @@ -252,7 +252,7 @@ struct MixProfile : virtual StoreCommand /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const Buildables & buildables); + void updateProfile(const DerivedPathsWithHints & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b68c5f6a7..f091ac186 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -285,9 +285,9 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -Buildable Installable::toBuildable() +DerivedPathWithHints Installable::toDerivedPathWithHints() { - auto buildables = toBuildables(); + auto buildables = toDerivedPathsWithHints(); if (buildables.size() != 1) throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); return std::move(buildables[0]); @@ -321,7 +321,7 @@ struct InstallableStorePath : Installable std::string what() override { return store->printStorePath(storePath); } - Buildables toBuildables() override + DerivedPathsWithHints toDerivedPathsWithHints() override { if (storePath.isDerivation()) { std::map> outputs; @@ -329,14 +329,14 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputsAndOptPaths(*store)) outputs.emplace(name, output.second); return { - BuildableFromDrv { + DerivedPathWithHintsBuilt { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - BuildableOpaque { + DerivedPathOpaque { .path = storePath, } }; @@ -349,9 +349,9 @@ struct InstallableStorePath : Installable } }; -Buildables InstallableValue::toBuildables() +DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() { - Buildables res; + DerivedPathsWithHints res; std::map>> drvsToOutputs; @@ -364,7 +364,7 @@ Buildables InstallableValue::toBuildables() } for (auto & i : drvsToOutputs) - res.push_back(BuildableFromDrv { i.first, i.second }); + res.push_back(DerivedPathWithHintsBuilt { i.first, i.second }); return res; } @@ -671,28 +671,28 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -Buildables build(ref store, Realise mode, +DerivedPathsWithHints build(ref store, Realise mode, std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; - Buildables buildables; + DerivedPathsWithHints buildables; - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & i : installables) { - for (auto & b : i->toBuildables()) { + for (auto & b : i->toDerivedPathsWithHints()) { std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { pathsToBuild.push_back(bo); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); pathsToBuild.push_back( - BuildableReqFromDrv{bfd.drvPath, outputNames}); + DerivedPath::Built{bfd.drvPath, outputNames}); }, }, b); buildables.push_back(std::move(b)); @@ -717,10 +717,10 @@ std::set toRealisedPaths( if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { res.insert(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { auto drv = store->readDerivation(bfd.drvPath); auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { @@ -751,8 +751,8 @@ std::set toRealisedPaths( settings.readOnlyMode = true; for (auto & i : installables) - for (auto & b : i->toBuildables()) - if (auto bfd = std::get_if(&b)) + for (auto & b : i->toDerivedPathsWithHints()) + if (auto bfd = std::get_if(&b)) res.insert(bfd->drvPath); } @@ -787,9 +787,9 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuildables()) + for (auto & b : i->toDerivedPathsWithHints()) std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -798,7 +798,7 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { drvPaths.insert(bfd.drvPath); }, }, b); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e5c6fe208..0bc932b52 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -29,9 +29,9 @@ struct Installable virtual std::string what() = 0; - virtual Buildables toBuildables() = 0; + virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0; - Buildable toBuildable(); + DerivedPathWithHints toDerivedPathWithHints(); App toApp(EvalState & state); @@ -74,7 +74,7 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; - Buildables toBuildables() override; + DerivedPathsWithHints toDerivedPathsWithHints() override; }; struct InstallableFlake : InstallableValue diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 24bc34b74..428adf4c2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { - std::vector drvs; + std::vector drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -56,8 +56,8 @@ void EvalState::realiseContext(const PathSet & context) /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; - std::vector buildReqs; - for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d }); + std::vector buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(buildReqs); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 20027e099..09af57871 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() } -void printMissing(ref store, const std::vector & paths, Verbosity lvl) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 18e0fb57d..9cb9e6da2 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -43,7 +43,7 @@ struct StorePathWithOutputs; void printMissing( ref store, - const std::vector & paths, + const std::vector & paths, Verbosity lvl = lvlInfo); void printMissing(ref store, const StorePathSet & willBuild, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8396abbcd..3ce538f77 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index fc6294545..732d4785d 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,17 +6,17 @@ namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) { Worker worker(*this); Goals goals; for (auto & br : reqs) { std::visit(overloaded { - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); }, - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); }, }, br.raw()); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6cc384719..7c1402918 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1191,20 +1191,20 @@ void LocalDerivationGoal::writeStructuredAttrs() } -static StorePath pathPartOfReq(const BuildableReq & req) +static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { return bo.path; }, - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { return bfd.drvPath; }, }, req.raw()); } -bool LocalDerivationGoal::isAllowed(const BuildableReq & req) +bool LocalDerivationGoal::isAllowed(const DerivedPath & req) { return this->isAllowed(pathPartOfReq(req)); } @@ -1332,7 +1332,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // an allowed derivation { throw Error("queryRealisation"); } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -1346,7 +1346,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo next->buildPaths(paths, buildMode); for (auto & path : paths) { - auto p = std::get_if(&path); + auto p = std::get_if(&path); if (!p) continue; auto & bfd = *p; auto outputs = next->queryDerivationOutputMap(bfd.drvPath); @@ -1380,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override { @@ -1388,7 +1388,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo client about what paths will be built/substituted or are already present. Probably not a big deal. */ - std::vector allowed; + std::vector allowed; for (auto & req : targets) { if (goal.isAllowed(req)) allowed.emplace_back(req); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index edb93f84e..d30be2351 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -116,7 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } - bool isAllowed(const BuildableReq & req); + bool isAllowed(const DerivedPath & req); friend struct RestrictedStore; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index fef4cb0cb..6c04d3ed3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; + std::vector topPaths; for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(BuildableOpaque{goal->storePath}); + topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 31fef2faa..a8c0c70b1 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -5,13 +5,13 @@ namespace nix { -nlohmann::json BuildableOpaque::toJSON(ref store) const { +nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { nlohmann::json res; res["path"] = store->printStorePath(path); return res; } -nlohmann::json BuildableFromDrv::toJSON(ref store) const { +nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { @@ -20,9 +20,9 @@ nlohmann::json BuildableFromDrv::toJSON(ref store) const { return res; } -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { +nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store) { auto res = nlohmann::json::array(); - for (const Buildable & buildable : buildables) { + for (const DerivedPathWithHints & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); }, buildable); @@ -31,17 +31,17 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) } -std::string BuildableOpaque::to_string(const Store & store) const { +std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } -std::string BuildableReqFromDrv::to_string(const Store & store) const { +std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); } -std::string BuildableReq::to_string(const Store & store) const +std::string DerivedPath::to_string(const Store & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, @@ -49,12 +49,12 @@ std::string BuildableReq::to_string(const Store & store) const } -BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) { return {store.parseStorePath(s)}; } -BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) +DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); assert(n != s.npos); @@ -66,12 +66,12 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_ return {drvPath, outputs}; } -BuildableReq BuildableReq::parse(const Store & store, std::string_view s) +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); return n == s.npos - ? (BuildableReq) BuildableOpaque::parse(store, s) - : (BuildableReq) BuildableReqFromDrv::parse(store, s); + ? (DerivedPath) DerivedPath::Opaque::parse(store, s) + : (DerivedPath) DerivedPath::Built::parse(store, s); } } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 8317f3995..0a0cf8105 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -12,54 +12,57 @@ namespace nix { class Store; -struct BuildableOpaque { +struct DerivedPathOpaque { StorePath path; nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; - static BuildableOpaque parse(const Store & store, std::string_view); + static DerivedPathOpaque parse(const Store & store, std::string_view); }; -struct BuildableReqFromDrv { +struct DerivedPathBuilt { StorePath drvPath; std::set outputs; std::string to_string(const Store & store) const; - static BuildableReqFromDrv parse(const Store & store, std::string_view); + static DerivedPathBuilt parse(const Store & store, std::string_view); }; -using _BuildableReqRaw = std::variant< - BuildableOpaque, - BuildableReqFromDrv +using _DerivedPathRaw = std::variant< + DerivedPathOpaque, + DerivedPathBuilt >; -struct BuildableReq : _BuildableReqRaw { - using Raw = _BuildableReqRaw; +struct DerivedPath : _DerivedPathRaw { + using Raw = _DerivedPathRaw; using Raw::Raw; + using Opaque = DerivedPathOpaque; + using Built = DerivedPathBuilt; + inline const Raw & raw() const { return static_cast(*this); } std::string to_string(const Store & store) const; - static BuildableReq parse(const Store & store, std::string_view); + static DerivedPath parse(const Store & store, std::string_view); }; -struct BuildableFromDrv { +struct DerivedPathWithHintsBuilt { StorePath drvPath; std::map> outputs; nlohmann::json toJSON(ref store) const; - static BuildableFromDrv parse(const Store & store, std::string_view); + static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); }; -using Buildable = std::variant< - BuildableOpaque, - BuildableFromDrv +using DerivedPathWithHints = std::variant< + DerivedPath::Opaque, + DerivedPathWithHintsBuilt >; -typedef std::vector Buildables; +typedef std::vector DerivedPathsWithHints; -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); +nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6b527dcb2..affd60472 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -260,14 +260,14 @@ static void writeValidPathInfo( } } -static std::vector readBuildableReqs(Store & store, unsigned int clientVersion, Source & from) +static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) { - std::vector reqs; + std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - reqs = worker_proto::read(store, from, Phantom> {}); + reqs = worker_proto::read(store, from, Phantom> {}); } else { for (auto & s : readStrings(from)) - reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq()); + reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); } return reqs; } @@ -506,7 +506,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopBuildPaths: { - auto drvs = readBuildableReqs(*store, clientVersion, from); + auto drvs = readDerivedPaths(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -870,7 +870,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryMissing: { - auto targets = readBuildableReqs(*store, clientVersion, from); + auto targets = readDerivedPaths(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1cb977be6..edaf75136 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -267,14 +267,14 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildPaths; Strings ss; for (auto & p : drvPaths) { - auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { [&](StorePathWithOutputs s) { ss.push_back(s.to_string(*this)); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index abfae1502..a99a2fc78 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -117,7 +117,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) return std::nullopt; } -void Store::queryMissing(const std::vector & targets, +void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, uint64_t & downloadSize_, uint64_t & narSize_) { @@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector & targets, Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { @@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); }; auto checkOutput = [&]( @@ -177,13 +177,13 @@ void Store::queryMissing(const std::vector & targets, drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, BuildableOpaque { path } )); + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { path } )); } } } }; - doPath = [&](const BuildableReq & req) { + doPath = [&](const DerivedPath & req) { { auto state(state_.lock()); @@ -191,7 +191,7 @@ void Store::queryMissing(const std::vector & targets, } std::visit(overloaded { - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); @@ -224,7 +224,7 @@ void Store::queryMissing(const std::vector & targets, mustBuildDrv(bfd.drvPath, *drv); }, - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { if (isValidPath(bo.path)) return; @@ -248,7 +248,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); }; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 2898b8d4f..865d64cf2 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -11,34 +11,34 @@ std::string StorePathWithOutputs::to_string(const Store & store) const } -BuildableReq StorePathWithOutputs::toBuildableReq() const +DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty() || path.isDerivation()) - return BuildableReqFromDrv { path, outputs }; + return DerivedPath::Built { path, outputs }; else - return BuildableOpaque { path }; + return DerivedPath::Opaque { path }; } -std::vector toBuildableReqs(const std::vector ss) +std::vector toDerivedPaths(const std::vector ss) { - std::vector reqs; - for (auto & s : ss) reqs.push_back(s.toBuildableReq()); + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toDerivedPath()); return reqs; } -std::variant StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p) +std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](BuildableOpaque bo) -> std::variant { + [&](DerivedPath::Opaque bo) -> std::variant { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](BuildableReqFromDrv bfd) -> std::variant { + [&](DerivedPath::Built bfd) -> std::variant { return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; }, }, p.raw()); diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 870cac08e..749348398 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -14,12 +14,12 @@ struct StorePathWithOutputs std::string to_string(const Store & store) const; - BuildableReq toBuildableReq() const; + DerivedPath toDerivedPath() const; - static std::variant tryFromBuildableReq(const BuildableReq &); + static std::variant tryFromDerivedPath(const DerivedPath &); }; -std::vector toBuildableReqs(const std::vector); +std::vector toDerivedPaths(const std::vector); std::pair parsePathWithOutputs(std::string_view s); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cb6402213..761b4a087 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -52,13 +52,13 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) } -BuildableReq read(const Store & store, Source & from, Phantom _) +DerivedPath read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return BuildableReq::parse(store, s); + return DerivedPath::parse(store, s); } -void write(const Store & store, Sink & out, const BuildableReq & req) +void write(const Store & store, Sink & out, const DerivedPath & req) { out << req.to_string(store); } @@ -670,14 +670,14 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & return {Realisation{.id = id, .outPath = *outPaths.begin()}}; } -static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { worker_proto::write(store, conn->to, reqs); } else { Strings ss; for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { [&](StorePathWithOutputs s) { ss.push_back(s.to_string(store)); @@ -694,12 +694,12 @@ static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, con } } -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeBuildableReqs(*this, conn, drvPaths); + writeDerivedPaths(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -838,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s } -void RemoteStore::queryMissing(const std::vector & targets, +void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { @@ -849,7 +849,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing; - writeBuildableReqs(*this, conn, targets); + writeDerivedPaths(*this, conn, targets); conn.processStderr(); willBuild = worker_proto::read(*this, conn->from, Phantom {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 20d366038..6cf76a46d 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -108,7 +108,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8b60bdc62..93fcb068f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath, void Store::substitutePaths(const StorePathSet & paths) { - std::vector paths2; + std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(BuildableOpaque{path}); + paths2.push_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p}); + std::vector subs; + for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 59d0983df..483f3c5fa 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -494,7 +494,7 @@ public: recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ virtual void buildPaths( - const std::vector & paths, + const std::vector & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an @@ -656,7 +656,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const std::vector & targets, + virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0255726ac..001ed25e3 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,7 +86,7 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); -MAKE_WORKER_PROTO(, BuildableReq); +MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6f8a61261..d46bc1f2b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -323,7 +323,7 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); auto buildPaths = [&](const std::vector & paths0) { - auto paths = toBuildableReqs(paths0); + auto paths = toDerivedPaths(paths0); /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index af1c69b87..e04954d45 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -419,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - std::vector targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)}); + targets.push_back(DerivedPath::Built{state.store->parseStorePath(drvPath)}); else - targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())}); + targets.push_back(DerivedPath::Opaque{state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -694,12 +694,12 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); - std::vector paths { + std::vector paths { (drv.queryDrvPath() != "") - ? (BuildableReq) (BuildableReqFromDrv { + ? (DerivedPath) (DerivedPath::Built { globals.state->store->parseStorePath(drv.queryDrvPath()) }) - : (BuildableReq) (BuildableOpaque { + : (DerivedPath) (DerivedPath::Opaque { globals.state->store->parseStorePath(drv.queryOutPath()) }), }; diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 0ccf960fb..5ceb2ae67 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -43,7 +43,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug(format("building user environment dependencies")); state.store->buildPaths( - toBuildableReqs(drvsToBuild), + toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ @@ -140,7 +140,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); state.store->buildPaths( - toBuildableReqs(topLevelDrvs), + toDerivedPaths(topLevelDrvs), state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 21c1e547b..b327793e7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -63,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path.toBuildableReq()}); + if (build) store->buildPaths({path.toDerivedPath()}); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -134,7 +134,7 @@ static void opRealise(Strings opFlags, Strings opArgs) uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing( - toBuildableReqs(paths), + toDerivedPaths(paths), willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { @@ -151,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(toBuildableReqs(paths), buildMode); + store->buildPaths(toDerivedPaths(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -882,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs) try { MonitorFdHup monitor(in.fd); - store->buildPaths(toBuildableReqs(paths)); + store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { assert(e.status); diff --git a/src/nix/build.cc b/src/nix/build.cc index 724ce9d79..0529ed382 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); for (auto & output : builtOutputs) { std::string symlink = outLink; @@ -80,7 +80,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile updateProfile(buildables); - if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump()); + if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); } }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index e86fbb3f7..53dccc63a 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand auto evalState = getEvalState(); auto app = installable->toApp(*evalState); - store->buildPaths(toBuildableReqs(app.context)); + store->buildPaths(toDerivedPaths(app.context)); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; @@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); - store->buildPaths({ BuildableReqFromDrv { drvPath } }); + store->buildPaths({ DerivedPath::Built { drvPath } }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 616e2073e..cae6ded40 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -160,7 +160,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) auto shellDrvPath = writeDerivation(*store, drv); /* Build the derivation. */ - store->buildPaths({BuildableReqFromDrv{shellDrvPath}}); + store->buildPaths({DerivedPath::Built{shellDrvPath}}); for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { auto & [_1, optPath] = outputAndOptPath; @@ -265,7 +265,7 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto buildable = installable->toBuildable(); + auto buildable = installable->toDerivedPathWithHints(); auto doRedirect = [&](const StorePath & path) { auto from = store->printStorePath(path); @@ -277,10 +277,10 @@ struct Common : InstallableCommand, MixProfile } }; std::visit(overloaded { - [&](const BuildableOpaque & bo) { + [&](const DerivedPathOpaque & bo) { doRedirect(bo.path); }, - [&](const BuildableFromDrv & bfd) { + [&](const DerivedPathWithHintsBuilt & bfd) { for (auto & [outputName, path] : bfd.outputs) if (path) doRedirect(*path); }, diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9d6d22a43..62a413e27 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -293,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::vector drvPaths; + std::vector drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { @@ -462,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back(BuildableReqFromDrv{drvPath}); + drvPaths.push_back(DerivedPath::Built{drvPath}); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index 67d3742d6..5010e3326 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -30,15 +30,15 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toBuildable(); + auto b = installable->toDerivedPathWithHints(); RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { return sub->getBuildLog(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { return sub->getBuildLog(bfd.drvPath); }, }, b); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b96e71844..ad824dd70 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast(installable)) { @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { @@ -259,16 +259,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}}); + pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}}); element.storePaths.insert(output.second); } }, @@ -391,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - std::vector pathsToBuild; + std::vector pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); @@ -426,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {"out"}}); // FIXME } } diff --git a/src/nix/run.cc b/src/nix/run.cc index 2e9bb41cc..ba60e57d8 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon auto app = installable->toApp(*state); - state->store->buildPaths(toBuildableReqs(app.context)); + state->store->buildPaths(toDerivedPaths(app.context)); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); From 179582872de60863fcabcf471f98930a25fd6df3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:05:21 -0400 Subject: [PATCH 08/10] Make `DerivedPathWithHints` a newtype This allows us to namespace its constructors under it. --- src/libcmd/command.cc | 6 +++--- src/libcmd/installables.cc | 26 +++++++++++++------------- src/libstore/buildable.cc | 4 ++-- src/libstore/buildable.hh | 15 ++++++++++++++- src/nix/build.cc | 6 +++--- src/nix/develop.cc | 6 +++--- src/nix/log.cc | 6 +++--- src/nix/profile.cc | 6 +++--- 8 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index dc1fbc43f..9da470c15 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -170,10 +170,10 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { result.push_back(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ @@ -181,7 +181,7 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) result.push_back(*output.second); } }, - }, buildable); + }, buildable.raw()); } if (result.size() != 1) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f091ac186..5d3026c1a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -329,14 +329,14 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputsAndOptPaths(*store)) outputs.emplace(name, output.second); return { - DerivedPathWithHintsBuilt { + DerivedPathWithHints::Built { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - DerivedPathOpaque { + DerivedPathWithHints::Opaque { .path = storePath, } }; @@ -364,7 +364,7 @@ DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() } for (auto & i : drvsToOutputs) - res.push_back(DerivedPathWithHintsBuilt { i.first, i.second }); + res.push_back(DerivedPathWithHints::Built { i.first, i.second }); return res; } @@ -684,17 +684,17 @@ DerivedPathsWithHints build(ref store, Realise mode, for (auto & i : installables) { for (auto & b : i->toDerivedPathsWithHints()) { std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { pathsToBuild.push_back(bo); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); pathsToBuild.push_back( DerivedPath::Built{bfd.drvPath, outputNames}); }, - }, b); + }, b.raw()); buildables.push_back(std::move(b)); } } @@ -717,10 +717,10 @@ std::set toRealisedPaths( if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { res.insert(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { auto drv = store->readDerivation(bfd.drvPath); auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { @@ -745,14 +745,14 @@ std::set toRealisedPaths( } } }, - }, b); + }, b.raw()); } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; for (auto & i : installables) for (auto & b : i->toDerivedPathsWithHints()) - if (auto bfd = std::get_if(&b)) + if (auto bfd = std::get_if(&b)) res.insert(bfd->drvPath); } @@ -789,7 +789,7 @@ StorePathSet toDerivations(ref store, for (auto & i : installables) for (auto & b : i->toDerivedPathsWithHints()) std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -798,10 +798,10 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { drvPaths.insert(bfd.drvPath); }, - }, b); + }, b.raw()); return drvPaths; } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index a8c0c70b1..eee38ba10 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,7 +11,7 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { return res; } -nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref store) const { +nlohmann::json DerivedPathWithHints::Built::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { @@ -25,7 +25,7 @@ nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildab for (const DerivedPathWithHints & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); - }, buildable); + }, buildable.raw()); } return res; } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 0a0cf8105..ce5ae5fc0 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -56,11 +56,24 @@ struct DerivedPathWithHintsBuilt { static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); }; -using DerivedPathWithHints = std::variant< +using _DerivedPathWithHintsRaw = std::variant< DerivedPath::Opaque, DerivedPathWithHintsBuilt >; +struct DerivedPathWithHints : _DerivedPathWithHintsRaw { + using Raw = _DerivedPathWithHintsRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = DerivedPathWithHintsBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + +}; + typedef std::vector DerivedPathsWithHints; nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); diff --git a/src/nix/build.cc b/src/nix/build.cc index 0529ed382..03159b6cc 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); for (auto & output : builtOutputs) { std::string symlink = outLink; @@ -75,7 +75,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildable); + }, buildable.raw()); } updateProfile(buildables); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index cae6ded40..7cc7b85be 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -277,14 +277,14 @@ struct Common : InstallableCommand, MixProfile } }; std::visit(overloaded { - [&](const DerivedPathOpaque & bo) { + [&](const DerivedPathWithHints::Opaque & bo) { doRedirect(bo.path); }, - [&](const DerivedPathWithHintsBuilt & bfd) { + [&](const DerivedPathWithHints::Built & bfd) { for (auto & [outputName, path] : bfd.outputs) if (path) doRedirect(*path); }, - }, buildable); + }, buildable.raw()); } return rewriteStrings(script, rewrites); diff --git a/src/nix/log.cc b/src/nix/log.cc index 5010e3326..638bb5073 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -35,13 +35,13 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { return sub->getBuildLog(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { return sub->getBuildLog(bfd.drvPath); }, - }, b); + }, b.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ad824dd70..667904cd2 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -259,11 +259,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? @@ -272,7 +272,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile element.storePaths.insert(output.second); } }, - }, buildable); + }, buildable.raw()); manifest.elements.emplace_back(std::move(element)); } From d8fa7517fad4272e20ff9b9b740c91158bc685e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:33:28 -0400 Subject: [PATCH 09/10] buildable.{cc,hh} -> derived-path.{cc,hh} --- src/libcmd/installables.hh | 2 +- src/libmain/shared.hh | 2 +- src/libstore/{buildable.cc => derived-path.cc} | 2 +- src/libstore/{buildable.hh => derived-path.hh} | 0 src/libstore/path-with-outputs.hh | 2 +- src/libstore/store-api.hh | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename src/libstore/{buildable.cc => derived-path.cc} (98%) rename src/libstore/{buildable.hh => derived-path.hh} (100%) diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 0bc932b52..403403c07 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "path.hh" #include "path-with-outputs.hh" -#include "buildable.hh" +#include "derived-path.hh" #include "eval.hh" #include "flake/flake.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 9cb9e6da2..05277d90a 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -4,7 +4,7 @@ #include "args.hh" #include "common-args.hh" #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" #include diff --git a/src/libstore/buildable.cc b/src/libstore/derived-path.cc similarity index 98% rename from src/libstore/buildable.cc rename to src/libstore/derived-path.cc index eee38ba10..13833c58e 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,4 @@ -#include "buildable.hh" +#include "derived-path.hh" #include "store-api.hh" #include diff --git a/src/libstore/buildable.hh b/src/libstore/derived-path.hh similarity index 100% rename from src/libstore/buildable.hh rename to src/libstore/derived-path.hh diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 749348398..4c4023dcb 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -3,7 +3,7 @@ #include #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 483f3c5fa..f66298991 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,7 +2,7 @@ #include "realisation.hh" #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" From 125a824228dbac0bb82023953f45318ea93e7ffa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:56:48 -0400 Subject: [PATCH 10/10] Document the derived path types. --- src/libstore/derived-path.hh | 50 +++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index ce5ae5fc0..7a2fe59de 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -2,7 +2,6 @@ #include "util.hh" #include "path.hh" -#include "path.hh" #include @@ -12,6 +11,13 @@ namespace nix { class Store; +/** + * An opaque derived path. + * + * Opaque derived paths are just store paths, and fully evaluated. They + * cannot be simplified further. Since they are opaque, they cannot be + * built, but they can fetched. + */ struct DerivedPathOpaque { StorePath path; @@ -20,6 +26,18 @@ struct DerivedPathOpaque { static DerivedPathOpaque parse(const Store & store, std::string_view); }; +/** + * A derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. + */ struct DerivedPathBuilt { StorePath drvPath; std::set outputs; @@ -33,6 +51,16 @@ using _DerivedPathRaw = std::variant< DerivedPathBuilt >; +/** + * A "derived path" is a very simple sort of expression that evaluates + * to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; using Raw::Raw; @@ -48,6 +76,11 @@ struct DerivedPath : _DerivedPathRaw { static DerivedPath parse(const Store & store, std::string_view); }; +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'DerivedPathWithHints' for more an explanation. + */ struct DerivedPathWithHintsBuilt { StorePath drvPath; std::map> outputs; @@ -61,6 +94,21 @@ using _DerivedPathWithHintsRaw = std::variant< DerivedPathWithHintsBuilt >; +/** + * A derived path with hints in the form of optional concrete output paths in the built case. + * + * This type is currently just used by the CLI. The paths are filled in + * during evaluation for derivations that know what paths they will + * produce in advanced, i.e. input-addressed or fixed-output content + * addressed derivations. + * + * That isn't very good, because it puts floating content-addressed + * derivations "at a disadvantage". It would be better to never rely on + * the output path of unbuilt derivations, and exclusively use the + * realizations types to work with built derivations' concrete output + * paths. + */ +// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd. struct DerivedPathWithHints : _DerivedPathWithHintsRaw { using Raw = _DerivedPathWithHintsRaw; using Raw::Raw;