Merge commit '9dfb97c987d8b9d6a3d15f016e40f22f91deb764' into path-info

This commit is contained in:
John Ericson 2021-04-05 18:30:02 -04:00
commit d5cef6c33a
35 changed files with 502 additions and 216 deletions

View file

@ -20,31 +20,6 @@
namespace nix { namespace nix {
nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
nlohmann::json res;
res["path"] = store->printStorePath(path);
return res;
}
nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = path ? store->printStorePath(*path) : "";
}
return res;
}
nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
auto res = nlohmann::json::array();
for (const Buildable & buildable : buildables) {
std::visit([&res, store](const auto & buildable) {
res.push_back(buildable.toJSON(store));
}, buildable);
}
return res;
}
void completeFlakeInputPath( void completeFlakeInputPath(
ref<EvalState> evalState, ref<EvalState> evalState,
const FlakeRef & flakeRef, const FlakeRef & flakeRef,
@ -704,19 +679,20 @@ Buildables build(ref<Store> store, Realise mode,
Buildables buildables; Buildables buildables;
std::vector<StorePathWithOutputs> pathsToBuild; std::vector<BuildableReq> pathsToBuild;
for (auto & i : installables) { for (auto & i : installables) {
for (auto & b : i->toBuildables()) { for (auto & b : i->toBuildables()) {
std::visit(overloaded { std::visit(overloaded {
[&](BuildableOpaque bo) { [&](BuildableOpaque bo) {
pathsToBuild.push_back({bo.path}); pathsToBuild.push_back(bo);
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
StringSet outputNames; StringSet outputNames;
for (auto & output : bfd.outputs) for (auto & output : bfd.outputs)
outputNames.insert(output.first); outputNames.insert(output.first);
pathsToBuild.push_back({bfd.drvPath, outputNames}); pathsToBuild.push_back(
BuildableReqFromDrv{bfd.drvPath, outputNames});
}, },
}, b); }, b);
buildables.push_back(std::move(b)); buildables.push_back(std::move(b));

View file

@ -2,13 +2,13 @@
#include "util.hh" #include "util.hh"
#include "path.hh" #include "path.hh"
#include "path-with-outputs.hh"
#include "buildable.hh"
#include "eval.hh" #include "eval.hh"
#include "flake/flake.hh" #include "flake/flake.hh"
#include <optional> #include <optional>
#include <nlohmann/json_fwd.hpp>
namespace nix { namespace nix {
struct DrvInfo; struct DrvInfo;
@ -16,25 +16,6 @@ struct SourceExprCommand;
namespace eval_cache { class EvalCache; class AttrCursor; } namespace eval_cache { class EvalCache; class AttrCursor; }
struct BuildableOpaque {
StorePath path;
nlohmann::json toJSON(ref<Store> store) const;
};
struct BuildableFromDrv {
StorePath drvPath;
std::map<std::string, std::optional<StorePath>> outputs;
nlohmann::json toJSON(ref<Store> store) const;
};
typedef std::variant<
BuildableOpaque,
BuildableFromDrv
> Buildable;
typedef std::vector<Buildable> Buildables;
nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
struct App struct App
{ {
std::vector<StorePathWithOutputs> context; std::vector<StorePathWithOutputs> context;

View file

@ -2,6 +2,7 @@
#include "util.hh" #include "util.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "store-api.hh" #include "store-api.hh"
#include "path-with-outputs.hh"
#include <cstring> #include <cstring>
#include <regex> #include <regex>
@ -19,7 +20,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs) DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("") : state(&state), attrs(nullptr), attrPath("")
{ {
auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
this->drvPath = store->printStorePath(drvPath); this->drvPath = store->printStorePath(drvPath);

View file

@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) :
void EvalState::realiseContext(const PathSet & context) void EvalState::realiseContext(const PathSet & context)
{ {
std::vector<StorePathWithOutputs> drvs; std::vector<BuildableReqFromDrv> drvs;
for (auto & i : context) { for (auto & i : context) {
auto [ctxS, outputName] = decodeContext(i); auto [ctxS, outputName] = decodeContext(i);
@ -43,7 +43,7 @@ void EvalState::realiseContext(const PathSet & context)
if (!store->isValidPath(ctx)) if (!store->isValidPath(ctx))
throw InvalidPathError(store->printStorePath(ctx)); throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) { if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); drvs.push_back({ctx, {outputName}});
} }
} }
@ -51,14 +51,16 @@ void EvalState::realiseContext(const PathSet & context)
if (!evalSettings.enableImportFromDerivation) if (!evalSettings.enableImportFromDerivation)
throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
store->printStorePath(drvs.begin()->path)); store->printStorePath(drvs.begin()->drvPath));
/* For performance, prefetch all substitute info. */ /* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown; StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize; uint64_t downloadSize, narSize;
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); std::vector<BuildableReq> buildReqs;
for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d });
store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs); store->buildPaths(buildReqs);
/* Add the output of this derivations to the allowed /* Add the output of this derivations to the allowed
paths. */ paths. */

View file

@ -36,7 +36,7 @@ void printGCWarning()
} }
void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl) void printMissing(ref<Store> store, const std::vector<BuildableReq> & paths, Verbosity lvl)
{ {
uint64_t downloadSize, narSize; uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown; StorePathSet willBuild, willSubstitute, unknown;

View file

@ -4,6 +4,7 @@
#include "args.hh" #include "args.hh"
#include "common-args.hh" #include "common-args.hh"
#include "path.hh" #include "path.hh"
#include "buildable.hh"
#include <signal.h> #include <signal.h>
@ -42,7 +43,7 @@ struct StorePathWithOutputs;
void printMissing( void printMissing(
ref<Store> store, ref<Store> store,
const std::vector<StorePathWithOutputs> & paths, const std::vector<BuildableReq> & paths,
Verbosity lvl = lvlInfo); Verbosity lvl = lvlInfo);
void printMissing(ref<Store> store, const StorePathSet & willBuild, void printMissing(ref<Store> store, const StorePathSet & willBuild,

View file

@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
state = &DerivationGoal::getDerivation; state = &DerivationGoal::getDerivation;
name = fmt( name = fmt(
"building of '%s' from .drv file", "building of '%s' from .drv file",
StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store));
trace("created"); trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
state = &DerivationGoal::haveDerivation; state = &DerivationGoal::haveDerivation;
name = fmt( name = fmt(
"building of '%s' from in-memory derivation", "building of '%s' from in-memory derivation",
StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store));
trace("created"); trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);

View file

@ -6,16 +6,20 @@
namespace nix { namespace nix {
void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) void Store::buildPaths(const std::vector<BuildableReq> & reqs, BuildMode buildMode)
{ {
Worker worker(*this); Worker worker(*this);
Goals goals; Goals goals;
for (auto & path : drvPaths) { for (auto & br : reqs) {
if (path.path.isDerivation()) std::visit(overloaded {
goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); [&](BuildableReqFromDrv bfd) {
else goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); },
[&](BuildableOpaque bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
} }
worker.run(goals); worker.run(goals);

View file

@ -1190,6 +1190,26 @@ void LocalDerivationGoal::writeStructuredAttrs()
chownToBuilder(tmpDir + "/.attrs.sh"); chownToBuilder(tmpDir + "/.attrs.sh");
} }
static StorePath pathPartOfReq(const BuildableReq & req)
{
return std::visit(overloaded {
[&](BuildableOpaque bo) {
return bo.path;
},
[&](BuildableReqFromDrv bfd) {
return bfd.drvPath;
},
}, req.raw());
}
bool LocalDerivationGoal::isAllowed(const BuildableReq & req)
{
return this->isAllowed(pathPartOfReq(req));
}
struct RestrictedStoreConfig : virtual LocalFSStoreConfig struct RestrictedStoreConfig : virtual LocalFSStoreConfig
{ {
using LocalFSStoreConfig::LocalFSStoreConfig; using LocalFSStoreConfig::LocalFSStoreConfig;
@ -1312,25 +1332,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
// an allowed derivation // an allowed derivation
{ throw Error("queryRealisation"); } { throw Error("queryRealisation"); }
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override
{ {
if (buildMode != bmNormal) throw Error("unsupported build mode"); if (buildMode != bmNormal) throw Error("unsupported build mode");
StorePathSet newPaths; StorePathSet newPaths;
for (auto & path : paths) { for (auto & req : paths) {
if (!goal.isAllowed(path.path)) if (!goal.isAllowed(req))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
} }
next->buildPaths(paths, buildMode); next->buildPaths(paths, buildMode);
for (auto & path : paths) { for (auto & path : paths) {
if (!path.path.isDerivation()) continue; auto p = std::get_if<BuildableReqFromDrv>(&path);
auto outputs = next->queryDerivationOutputMap(path.path); if (!p) continue;
for (auto & output : outputs) auto & bfd = *p;
if (wantOutput(output.first, path.outputs)) auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
newPaths.insert(output.second); for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs))
newPaths.insert(outputPath);
} }
StorePathSet closure; StorePathSet closure;
@ -1358,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
void addSignatures(const StorePath & storePath, const StringSet & sigs) override void addSignatures(const StorePath & storePath, const StringSet & sigs) override
{ unsupported("addSignatures"); } { unsupported("addSignatures"); }
void queryMissing(const std::vector<StorePathWithOutputs> & targets, void queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override uint64_t & downloadSize, uint64_t & narSize) override
{ {
@ -1366,12 +1388,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
client about what paths will be built/substituted or are client about what paths will be built/substituted or are
already present. Probably not a big deal. */ already present. Probably not a big deal. */
std::vector<StorePathWithOutputs> allowed; std::vector<BuildableReq> allowed;
for (auto & path : targets) { for (auto & req : targets) {
if (goal.isAllowed(path.path)) if (goal.isAllowed(req))
allowed.emplace_back(path); allowed.emplace_back(req);
else else
unknown.insert(path.path); unknown.insert(pathPartOfReq(req));
} }
next->queryMissing(allowed, willBuild, willSubstitute, next->queryMissing(allowed, willBuild, willSubstitute,

View file

@ -116,6 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal
{ {
return inputPaths.count(path) || addedPaths.count(path); return inputPaths.count(path) || addedPaths.count(path);
} }
bool isAllowed(const BuildableReq & req);
friend struct RestrictedStore; friend struct RestrictedStore;

View file

@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal)
void Worker::run(const Goals & _topGoals) void Worker::run(const Goals & _topGoals)
{ {
std::vector<nix::StorePathWithOutputs> topPaths; std::vector<nix::BuildableReq> topPaths;
for (auto & i : _topGoals) { for (auto & i : _topGoals) {
topGoals.insert(i); topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) { if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back({goal->drvPath, goal->wantedOutputs}); topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs});
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) { } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back({goal->storePath}); topPaths.push_back(BuildableOpaque{goal->storePath});
} }
} }

77
src/libstore/buildable.cc Normal file
View file

@ -0,0 +1,77 @@
#include "buildable.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix {
nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
nlohmann::json res;
res["path"] = store->printStorePath(path);
return res;
}
nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = path ? store->printStorePath(*path) : "";
}
return res;
}
nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
auto res = nlohmann::json::array();
for (const Buildable & buildable : buildables) {
std::visit([&res, store](const auto & buildable) {
res.push_back(buildable.toJSON(store));
}, buildable);
}
return res;
}
std::string BuildableOpaque::to_string(const Store & store) const {
return store.printStorePath(path);
}
std::string BuildableReqFromDrv::to_string(const Store & store) const {
return store.printStorePath(drvPath)
+ "!"
+ (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
}
std::string BuildableReq::to_string(const Store & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
this->raw());
}
BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s)
{
return {store.parseStorePath(s)};
}
BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s)
{
size_t n = s.find("!");
assert(n != s.npos);
auto drvPath = store.parseStorePath(s.substr(0, n));
auto outputsS = s.substr(n + 1);
std::set<string> outputs;
if (outputsS != "*")
outputs = tokenizeString<std::set<string>>(outputsS);
return {drvPath, outputs};
}
BuildableReq BuildableReq::parse(const Store & store, std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? (BuildableReq) BuildableOpaque::parse(store, s)
: (BuildableReq) BuildableReqFromDrv::parse(store, s);
}
}

65
src/libstore/buildable.hh Normal file
View file

@ -0,0 +1,65 @@
#pragma once
#include "util.hh"
#include "path.hh"
#include <optional>
#include <variant>
#include <nlohmann/json_fwd.hpp>
namespace nix {
class Store;
struct BuildableOpaque {
StorePath path;
nlohmann::json toJSON(ref<Store> store) const;
std::string to_string(const Store & store) const;
static BuildableOpaque parse(const Store & store, std::string_view);
};
struct BuildableReqFromDrv {
StorePath drvPath;
std::set<std::string> outputs;
std::string to_string(const Store & store) const;
static BuildableReqFromDrv parse(const Store & store, std::string_view);
};
using _BuildableReqRaw = std::variant<
BuildableOpaque,
BuildableReqFromDrv
>;
struct BuildableReq : _BuildableReqRaw {
using Raw = _BuildableReqRaw;
using Raw::Raw;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
std::string to_string(const Store & store) const;
static BuildableReq parse(const Store & store, std::string_view);
};
struct BuildableFromDrv {
StorePath drvPath;
std::map<std::string, std::optional<StorePath>> outputs;
nlohmann::json toJSON(ref<Store> store) const;
static BuildableFromDrv parse(const Store & store, std::string_view);
};
using Buildable = std::variant<
BuildableOpaque,
BuildableFromDrv
>;
typedef std::vector<Buildable> Buildables;
nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
}

View file

@ -2,6 +2,7 @@
#include "monitor-fd.hh" #include "monitor-fd.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "store-api.hh" #include "store-api.hh"
#include "path-with-outputs.hh"
#include "finally.hh" #include "finally.hh"
#include "affinity.hh" #include "affinity.hh"
#include "archive.hh" #include "archive.hh"
@ -259,6 +260,18 @@ static void writeValidPathInfo(
} }
} }
static std::vector<BuildableReq> readBuildableReqs(Store & store, unsigned int clientVersion, Source & from)
{
std::vector<BuildableReq> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
reqs = worker_proto::read(store, from, Phantom<std::vector<BuildableReq>> {});
} else {
for (auto & s : readStrings<Strings>(from))
reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq());
}
return reqs;
}
static void performOp(TunnelLogger * logger, ref<Store> store, static void performOp(TunnelLogger * logger, ref<Store> store,
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
Source & from, BufferedSink & to, unsigned int op) Source & from, BufferedSink & to, unsigned int op)
@ -493,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case wopBuildPaths: { case wopBuildPaths: {
std::vector<StorePathWithOutputs> drvs; auto drvs = readBuildableReqs(*store, clientVersion, from);
for (auto & s : readStrings<Strings>(from))
drvs.push_back(store->parsePathWithOutputs(s));
BuildMode mode = bmNormal; BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from); mode = (BuildMode) readInt(from);
@ -859,9 +870,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case wopQueryMissing: { case wopQueryMissing: {
std::vector<StorePathWithOutputs> targets; auto targets = readBuildableReqs(*store, clientVersion, from);
for (auto & s : readStrings<Strings>(from))
targets.push_back(store->parsePathWithOutputs(s));
logger->startWork(); logger->startWork();
StorePathSet willBuild, willSubstitute, unknown; StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize; uint64_t downloadSize, narSize;

View file

@ -590,14 +590,6 @@ std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& d
} }
std::string StorePathWithOutputs::to_string(const Store & store) const
{
return outputs.empty()
? store.printStorePath(path)
: store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
}
bool wantOutput(const string & output, const std::set<string> & wanted) bool wantOutput(const string & output, const std::set<string> & wanted)
{ {
return wanted.empty() || wanted.find(output) != wanted.end(); return wanted.empty() || wanted.find(output) != wanted.end();

View file

@ -3,6 +3,7 @@
#include "remote-store.hh" #include "remote-store.hh"
#include "serve-protocol.hh" #include "serve-protocol.hh"
#include "store-api.hh" #include "store-api.hh"
#include "path-with-outputs.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "ssh.hh" #include "ssh.hh"
#include "derivations.hh" #include "derivations.hh"
@ -265,14 +266,23 @@ public:
return status; return status;
} }
void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override void buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode) override
{ {
auto conn(connections->get()); auto conn(connections->get());
conn->to << cmdBuildPaths; conn->to << cmdBuildPaths;
Strings ss; Strings ss;
for (auto & p : drvPaths) for (auto & p : drvPaths) {
ss.push_back(p.to_string(*this)); auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
std::visit(overloaded {
[&](StorePathWithOutputs s) {
ss.push_back(s.to_string(*this));
},
[&](StorePath drvPath) {
throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
},
}, sOrDrvPath);
}
conn->to << ss; conn->to << ss;
putBuildSettings(*conn); putBuildSettings(*conn);

View file

@ -116,7 +116,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
return std::nullopt; return std::nullopt;
} }
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, void Store::queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
uint64_t & downloadSize_, uint64_t & narSize_) uint64_t & downloadSize_, uint64_t & narSize_)
{ {
@ -144,7 +144,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
std::function<void(StorePathWithOutputs)> doPath; std::function<void(BuildableReq)> doPath;
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
{ {
@ -153,7 +153,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
} }
for (auto & i : drv.inputDrvs) for (auto & i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second }));
}; };
auto checkOutput = [&]( auto checkOutput = [&](
@ -176,24 +176,25 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
drvState->outPaths.insert(outPath); drvState->outPaths.insert(outPath);
if (!drvState->left) { if (!drvState->left) {
for (auto & path : drvState->outPaths) for (auto & path : drvState->outPaths)
pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); pool.enqueue(std::bind(doPath, BuildableOpaque { path } ));
} }
} }
} }
}; };
doPath = [&](const StorePathWithOutputs & path) { doPath = [&](const BuildableReq & req) {
{ {
auto state(state_.lock()); auto state(state_.lock());
if (!state->done.insert(path.to_string(*this)).second) return; if (!state->done.insert(req.to_string(*this)).second) return;
} }
if (path.path.isDerivation()) { std::visit(overloaded {
if (!isValidPath(path.path)) { [&](BuildableReqFromDrv bfd) {
if (!isValidPath(bfd.drvPath)) {
// FIXME: we could try to substitute the derivation. // FIXME: we could try to substitute the derivation.
auto state(state_.lock()); auto state(state_.lock());
state->unknown.insert(path.path); state->unknown.insert(bfd.drvPath);
return; return;
} }
@ -201,52 +202,54 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
/* true for regular derivations, and CA derivations for which we /* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */ have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true; auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) { for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) {
if (!pathOpt) { if (!pathOpt) {
knownOutputPaths = false; knownOutputPaths = false;
break; break;
} }
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt))
invalid.insert(*pathOpt); invalid.insert(*pathOpt);
} }
if (knownOutputPaths && invalid.empty()) return; if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(path.path)); auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath));
ParsedDerivation parsedDrv(StorePath(path.path), *drv); ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv);
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid) for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState)); pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState));
} else } else
mustBuildDrv(path.path, *drv); mustBuildDrv(bfd.drvPath, *drv);
} else { },
[&](BuildableOpaque bo) {
if (isValidPath(path.path)) return; if (isValidPath(bo.path)) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos);
if (infos.empty()) { if (infos.empty()) {
auto state(state_.lock()); auto state(state_.lock());
state->unknown.insert(path.path); state->unknown.insert(bo.path);
return; return;
} }
auto info = infos.find(path.path); auto info = infos.find(bo.path);
assert(info != infos.end()); assert(info != infos.end());
{ {
auto state(state_.lock()); auto state(state_.lock());
state->willSubstitute.insert(path.path); state->willSubstitute.insert(bo.path);
state->downloadSize += info->second.downloadSize; state->downloadSize += info->second.downloadSize;
state->narSize += info->second.narSize; state->narSize += info->second.narSize;
} }
for (auto & ref : info->second.references) for (auto & ref : info->second.references)
pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); pool.enqueue(std::bind(doPath, BuildableOpaque { ref }));
} },
}, req.raw());
}; };
for (auto & path : targets) for (auto & path : targets)

View file

@ -0,0 +1,71 @@
#include "path-with-outputs.hh"
#include "store-api.hh"
namespace nix {
std::string StorePathWithOutputs::to_string(const Store & store) const
{
return outputs.empty()
? store.printStorePath(path)
: store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
}
BuildableReq StorePathWithOutputs::toBuildableReq() const
{
if (!outputs.empty() || path.isDerivation())
return BuildableReqFromDrv { path, outputs };
else
return BuildableOpaque { path };
}
std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs> ss)
{
std::vector<BuildableReq> reqs;
for (auto & s : ss) reqs.push_back(s.toBuildableReq());
return reqs;
}
std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p)
{
return std::visit(overloaded {
[&](BuildableOpaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
if (bo.path.isDerivation()) {
// drv path gets interpreted as "build", not "get drv file itself"
return bo.path;
}
return StorePathWithOutputs { bo.path };
},
[&](BuildableReqFromDrv bfd) -> std::variant<StorePathWithOutputs, StorePath> {
return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
},
}, p.raw());
}
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
}
StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
{
auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) };
}
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
{
auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
}
}

View file

@ -0,0 +1,35 @@
#pragma once
#include <variant>
#include "path.hh"
#include "buildable.hh"
namespace nix {
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
std::string to_string(const Store & store) const;
BuildableReq toBuildableReq() const;
static std::variant<StorePathWithOutputs, StorePath> tryFromBuildableReq(const BuildableReq &);
};
std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs>);
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
class Store;
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
}

View file

@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const
return res; return res;
} }
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
}
StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
{
auto [path, outputs] = nix::parsePathWithOutputs(s);
return {parseStorePath(path), std::move(outputs)};
}
} }

View file

@ -68,16 +68,6 @@ typedef std::map<string, StorePath> OutputPathMap;
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv"; const std::string drvExtension = ".drv";
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
std::string to_string(const Store & store) const;
};
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
} }
namespace std { namespace std {

View file

@ -1,5 +1,6 @@
#include "serialise.hh" #include "serialise.hh"
#include "util.hh" #include "util.hh"
#include "path-with-outputs.hh"
#include "remote-fs-accessor.hh" #include "remote-fs-accessor.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
@ -50,6 +51,19 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
out << renderContentAddress(ca); out << renderContentAddress(ca);
} }
BuildableReq read(const Store & store, Source & from, Phantom<BuildableReq> _)
{
auto s = readString(from);
return BuildableReq::parse(store, s);
}
void write(const Store & store, Sink & out, const BuildableReq & req)
{
out << req.to_string(store);
}
Realisation read(const Store & store, Source & from, Phantom<Realisation> _) Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
{ {
std::string rawInput = readString(from); std::string rawInput = readString(from);
@ -58,8 +72,12 @@ Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
"remote-protocol" "remote-protocol"
); );
} }
void write(const Store & store, Sink & out, const Realisation & realisation) void write(const Store & store, Sink & out, const Realisation & realisation)
{ out << realisation.toJSON().dump(); } {
out << realisation.toJSON().dump();
}
DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _) DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
{ {
@ -653,16 +671,36 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
return {Realisation{.id = id, .outPath = *outPaths.begin()}}; return {Realisation{.id = id, .outPath = *outPaths.begin()}};
} }
static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector<BuildableReq> & reqs)
{
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
worker_proto::write(store, conn->to, reqs);
} else {
Strings ss;
for (auto & p : reqs) {
auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
std::visit(overloaded {
[&](StorePathWithOutputs s) {
ss.push_back(s.to_string(store));
},
[&](StorePath drvPath) {
throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
store.printStorePath(drvPath),
GET_PROTOCOL_MAJOR(conn->daemonVersion),
GET_PROTOCOL_MINOR(conn->daemonVersion));
},
}, sOrDrvPath);
}
conn->to << ss;
}
}
void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) void RemoteStore::buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode)
{ {
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopBuildPaths; conn->to << wopBuildPaths;
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
Strings ss; writeBuildableReqs(*this, conn, drvPaths);
for (auto & p : drvPaths)
ss.push_back(p.to_string(*this));
conn->to << ss;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
conn->to << buildMode; conn->to << buildMode;
else else
@ -801,7 +839,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
} }
void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets, void RemoteStore::queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) uint64_t & downloadSize, uint64_t & narSize)
{ {
@ -812,10 +850,7 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
// to prevent a deadlock. // to prevent a deadlock.
goto fallback; goto fallback;
conn->to << wopQueryMissing; conn->to << wopQueryMissing;
Strings ss; writeBuildableReqs(*this, conn, targets);
for (auto & p : targets)
ss.push_back(p.to_string(*this));
conn->to << ss;
conn.processStderr(); conn.processStderr();
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});

View file

@ -85,7 +85,7 @@ public:
std::optional<const Realisation> queryRealisation(const DrvOutput &) override; std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override; void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override; BuildMode buildMode) override;
@ -108,7 +108,7 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
void queryMissing(const std::vector<StorePathWithOutputs> & targets, void queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override; uint64_t & downloadSize, uint64_t & narSize) override;

View file

@ -54,13 +54,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
} }
StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
{
auto [path2, outputs] = nix::parsePathWithOutputs(path);
return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
}
/* Store paths have the following form: /* Store paths have the following form:
<realized-path> = <store>/<h>-<name> <realized-path> = <store>/<h>-<name>
@ -552,10 +545,10 @@ void Store::queryPathInfo(const StorePath & storePath,
void Store::substitutePaths(const StorePathSet & paths) void Store::substitutePaths(const StorePathSet & paths)
{ {
std::vector<StorePathWithOutputs> paths2; std::vector<BuildableReq> paths2;
for (auto & path : paths) for (auto & path : paths)
if (!path.isDerivation()) if (!path.isDerivation())
paths2.push_back({path}); paths2.push_back(BuildableOpaque{path});
uint64_t downloadSize, narSize; uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown; StorePathSet willBuild, willSubstitute, unknown;
queryMissing(paths2, queryMissing(paths2,
@ -563,8 +556,8 @@ void Store::substitutePaths(const StorePathSet & paths)
if (!willSubstitute.empty()) if (!willSubstitute.empty())
try { try {
std::vector<StorePathWithOutputs> subs; std::vector<BuildableReq> subs;
for (auto & p : willSubstitute) subs.push_back({p}); for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p});
buildPaths(subs); buildPaths(subs);
} catch (Error & e) { } catch (Error & e) {
logWarning(e.info()); logWarning(e.info());

View file

@ -2,6 +2,7 @@
#include "realisation.hh" #include "realisation.hh"
#include "path.hh" #include "path.hh"
#include "buildable.hh"
#include "hash.hh" #include "hash.hh"
#include "content-address.hh" #include "content-address.hh"
#include "serialise.hh" #include "serialise.hh"
@ -263,11 +264,6 @@ public:
PathSet printStorePathSet(const StorePathSet & path) const; PathSet printStorePathSet(const StorePathSet & path) const;
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
StorePathWithOutputs parsePathWithOutputs(const string & s);
/* Display a set of paths in human-readable form (i.e., between quotes /* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */ and separated by commas). */
std::string showPaths(const StorePathSet & paths); std::string showPaths(const StorePathSet & paths);
@ -291,8 +287,6 @@ public:
result. */ result. */
StorePath followLinksToStorePath(std::string_view path) const; StorePath followLinksToStorePath(std::string_view path) const;
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
/* Constructs a unique store path name. */ /* Constructs a unique store path name. */
StorePath makeStorePath(std::string_view type, StorePath makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const; std::string_view hash, std::string_view name) const;
@ -496,7 +490,7 @@ public:
recursively building any sub-derivations. For inputs that are recursively building any sub-derivations. For inputs that are
not derivations, substitute them. */ not derivations, substitute them. */
virtual void buildPaths( virtual void buildPaths(
const std::vector<StorePathWithOutputs> & paths, const std::vector<BuildableReq> & paths,
BuildMode buildMode = bmNormal); BuildMode buildMode = bmNormal);
/* Build a single non-materialized derivation (i.e. not from an /* Build a single non-materialized derivation (i.e. not from an
@ -658,7 +652,7 @@ public:
/* Given a set of paths that are to be built, return the set of /* Given a set of paths that are to be built, return the set of
derivations that will be built, and the set of output paths derivations that will be built, and the set of output paths
that will be substituted. */ that will be substituted. */
virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets, virtual void queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize); uint64_t & downloadSize, uint64_t & narSize);

View file

@ -86,9 +86,11 @@ namespace worker_proto {
MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, std::string);
MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, StorePath);
MAKE_WORKER_PROTO(, ContentAddress); MAKE_WORKER_PROTO(, ContentAddress);
MAKE_WORKER_PROTO(, BuildableReq);
MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>); MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
#define X_ template<typename K, typename V> #define X_ template<typename K, typename V>
@ -113,6 +115,26 @@ MAKE_WORKER_PROTO(X_, Y_);
MAKE_WORKER_PROTO(, std::optional<StorePath>); MAKE_WORKER_PROTO(, std::optional<StorePath>);
MAKE_WORKER_PROTO(, std::optional<ContentAddress>); MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
template<typename T>
std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
{
std::vector<T> resSet;
auto size = readNum<size_t>(from);
while (size--) {
resSet.push_back(read(store, from, Phantom<T> {}));
}
return resSet;
}
template<typename T>
void write(const Store & store, Sink & out, const std::vector<T> & resSet)
{
out << resSet.size();
for (auto & key : resSet) {
write(store, out, key);
}
}
template<typename T> template<typename T>
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _) std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
{ {

View file

@ -12,6 +12,7 @@
#include "affinity.hh" #include "affinity.hh"
#include "util.hh" #include "util.hh"
#include "shared.hh" #include "shared.hh"
#include "path-with-outputs.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "get-drvs.hh" #include "get-drvs.hh"
@ -321,7 +322,8 @@ static void main_nix_build(int argc, char * * argv)
state->printStats(); state->printStats();
auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) { auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths0) {
auto paths = toBuildableReqs(paths0);
/* Note: we do this even when !printMissing to efficiently /* Note: we do this even when !printMissing to efficiently
fetch binary cache data. */ fetch binary cache data. */
uint64_t downloadSize, narSize; uint64_t downloadSize, narSize;

View file

@ -6,6 +6,7 @@
#include "globals.hh" #include "globals.hh"
#include "names.hh" #include "names.hh"
#include "profiles.hh" #include "profiles.hh"
#include "path-with-outputs.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "local-fs-store.hh" #include "local-fs-store.hh"
@ -418,13 +419,13 @@ static void queryInstSources(EvalState & state,
static void printMissing(EvalState & state, DrvInfos & elems) static void printMissing(EvalState & state, DrvInfos & elems)
{ {
std::vector<StorePathWithOutputs> targets; std::vector<BuildableReq> targets;
for (auto & i : elems) { for (auto & i : elems) {
Path drvPath = i.queryDrvPath(); Path drvPath = i.queryDrvPath();
if (drvPath != "") if (drvPath != "")
targets.push_back({state.store->parseStorePath(drvPath)}); targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)});
else else
targets.push_back({state.store->parseStorePath(i.queryOutPath())}); targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())});
} }
printMissing(state.store, targets); printMissing(state.store, targets);
@ -693,17 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
if (globals.forceName != "") if (globals.forceName != "")
drv.setName(globals.forceName); drv.setName(globals.forceName);
if (drv.queryDrvPath() != "") { std::vector<BuildableReq> paths {
std::vector<StorePathWithOutputs> paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}}; (drv.queryDrvPath() != "")
printMissing(globals.state->store, paths); ? (BuildableReq) (BuildableReqFromDrv {
if (globals.dryRun) return; globals.state->store->parseStorePath(drv.queryDrvPath())
globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); })
} else { : (BuildableReq) (BuildableOpaque {
printMissing(globals.state->store, globals.state->store->parseStorePath(drv.queryOutPath())
{{globals.state->store->parseStorePath(drv.queryOutPath())}}); }),
if (globals.dryRun) return; };
globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); printMissing(globals.state->store, paths);
} if (globals.dryRun) return;
globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal);
debug(format("switching to new user environment")); debug(format("switching to new user environment"));
Path generation = createGeneration( Path generation = createGeneration(

View file

@ -2,6 +2,7 @@
#include "util.hh" #include "util.hh"
#include "derivations.hh" #include "derivations.hh"
#include "store-api.hh" #include "store-api.hh"
#include "path-with-outputs.hh"
#include "local-fs-store.hh" #include "local-fs-store.hh"
#include "globals.hh" #include "globals.hh"
#include "shared.hh" #include "shared.hh"
@ -41,7 +42,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())});
debug(format("building user environment dependencies")); debug(format("building user environment dependencies"));
state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); state.store->buildPaths(
toBuildableReqs(drvsToBuild),
state.repair ? bmRepair : bmNormal);
/* Construct the whole top level derivation. */ /* Construct the whole top level derivation. */
StorePathSet references; StorePathSet references;
@ -136,7 +139,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
debug("building user environment"); debug("building user environment");
std::vector<StorePathWithOutputs> topLevelDrvs; std::vector<StorePathWithOutputs> topLevelDrvs;
topLevelDrvs.push_back({topLevelDrv}); topLevelDrvs.push_back({topLevelDrv});
state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); state.store->buildPaths(
toBuildableReqs(topLevelDrvs),
state.repair ? bmRepair : bmNormal);
/* Switch the current user environment to the output path. */ /* Switch the current user environment to the output path. */
auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();

View file

@ -10,6 +10,7 @@
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "graphml.hh" #include "graphml.hh"
#include "legacy.hh" #include "legacy.hh"
#include "path-with-outputs.hh"
#include <iostream> #include <iostream>
#include <algorithm> #include <algorithm>
@ -62,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store); auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
if (path.path.isDerivation()) { if (path.path.isDerivation()) {
if (build) store->buildPaths({path}); if (build) store->buildPaths({path.toBuildableReq()});
auto outputPaths = store->queryDerivationOutputMap(path.path); auto outputPaths = store->queryDerivationOutputMap(path.path);
Derivation drv = store->derivationFromPath(path.path); Derivation drv = store->derivationFromPath(path.path);
rootNr++; rootNr++;
@ -128,11 +129,13 @@ static void opRealise(Strings opFlags, Strings opArgs)
std::vector<StorePathWithOutputs> paths; std::vector<StorePathWithOutputs> paths;
for (auto & i : opArgs) for (auto & i : opArgs)
paths.push_back(store->followLinksToStorePathWithOutputs(i)); paths.push_back(followLinksToStorePathWithOutputs(*store, i));
uint64_t downloadSize, narSize; uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown; StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); store->queryMissing(
toBuildableReqs(paths),
willBuild, willSubstitute, unknown, downloadSize, narSize);
if (ignoreUnknown) { if (ignoreUnknown) {
std::vector<StorePathWithOutputs> paths2; std::vector<StorePathWithOutputs> paths2;
@ -148,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
if (dryRun) return; if (dryRun) return;
/* Build all paths at the same time to exploit parallelism. */ /* Build all paths at the same time to exploit parallelism. */
store->buildPaths(paths, buildMode); store->buildPaths(toBuildableReqs(paths), buildMode);
if (!ignoreUnknown) if (!ignoreUnknown)
for (auto & i : paths) { for (auto & i : paths) {
@ -879,13 +882,13 @@ static void opServe(Strings opFlags, Strings opArgs)
std::vector<StorePathWithOutputs> paths; std::vector<StorePathWithOutputs> paths;
for (auto & s : readStrings<Strings>(in)) for (auto & s : readStrings<Strings>(in))
paths.push_back(store->parsePathWithOutputs(s)); paths.push_back(parsePathWithOutputs(*store, s));
getBuildSettings(); getBuildSettings();
try { try {
MonitorFdHup monitor(in.fd); MonitorFdHup monitor(in.fd);
store->buildPaths(paths); store->buildPaths(toBuildableReqs(paths));
out << 0; out << 0;
} catch (Error & e) { } catch (Error & e) {
assert(e.status); assert(e.status);

View file

@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand
auto evalState = getEvalState(); auto evalState = getEvalState();
auto app = installable->toApp(*evalState); auto app = installable->toApp(*evalState);
store->buildPaths(app.context); store->buildPaths(toBuildableReqs(app.context));
auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
const flake::LockFlags lockFlags{ .writeLockFile = false }; const flake::LockFlags lockFlags{ .writeLockFile = false };
@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand
StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
store->buildPaths({{drvPath}}); store->buildPaths({ BuildableReqFromDrv { drvPath } });
auto outPathS = store->printStorePath(outPath); auto outPathS = store->printStorePath(outPath);

View file

@ -3,6 +3,7 @@
#include "common-args.hh" #include "common-args.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "path-with-outputs.hh"
#include "derivations.hh" #include "derivations.hh"
#include "affinity.hh" #include "affinity.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
@ -159,7 +160,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
auto shellDrvPath = writeDerivation(*store, drv); auto shellDrvPath = writeDerivation(*store, drv);
/* Build the derivation. */ /* Build the derivation. */
store->buildPaths({{shellDrvPath}}); store->buildPaths({BuildableReqFromDrv{shellDrvPath}});
for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
auto & [_1, optPath] = outputAndOptPath; auto & [_1, optPath] = outputAndOptPath;

View file

@ -7,6 +7,7 @@
#include "get-drvs.hh" #include "get-drvs.hh"
#include "store-api.hh" #include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"
#include "path-with-outputs.hh"
#include "attr-path.hh" #include "attr-path.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "registry.hh" #include "registry.hh"
@ -292,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand
} }
}; };
std::vector<StorePathWithOutputs> drvPaths; std::vector<BuildableReq> drvPaths;
auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) {
try { try {
@ -461,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand
fmt("%s.%s.%s", name, attr.name, attr2.name), fmt("%s.%s.%s", name, attr.name, attr2.name),
*attr2.value, *attr2.pos); *attr2.value, *attr2.pos);
if ((std::string) attr.name == settings.thisSystem.get()) if ((std::string) attr.name == settings.thisSystem.get())
drvPaths.push_back({drvPath}); drvPaths.push_back(BuildableReqFromDrv{drvPath});
} }
} }
} }

View file

@ -242,7 +242,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
{ {
ProfileManifest manifest(*getEvalState(), *profile); ProfileManifest manifest(*getEvalState(), *profile);
std::vector<StorePathWithOutputs> pathsToBuild; std::vector<BuildableReq> pathsToBuild;
for (auto & installable : installables) { for (auto & installable : installables) {
if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
@ -258,7 +258,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
attrPath, attrPath,
}; };
pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}}); pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}});
manifest.elements.emplace_back(std::move(element)); manifest.elements.emplace_back(std::move(element));
} else { } else {
@ -269,12 +269,15 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
std::visit(overloaded { std::visit(overloaded {
[&](BuildableOpaque bo) { [&](BuildableOpaque bo) {
pathsToBuild.push_back({bo.path, {}}); pathsToBuild.push_back(bo);
element.storePaths.insert(bo.path); element.storePaths.insert(bo.path);
}, },
[&](BuildableFromDrv bfd) { [&](BuildableFromDrv bfd) {
// TODO: Why are we querying if we know the output
// names already? Is it just to figure out what the
// default one is?
for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
pathsToBuild.push_back({bfd.drvPath, {output.first}}); pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}});
element.storePaths.insert(output.second); element.storePaths.insert(output.second);
} }
}, },
@ -397,7 +400,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
auto matchers = getMatchers(store); auto matchers = getMatchers(store);
// FIXME: code duplication // FIXME: code duplication
std::vector<StorePathWithOutputs> pathsToBuild; std::vector<BuildableReq> pathsToBuild;
for (size_t i = 0; i < manifest.elements.size(); ++i) { for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]); auto & element(manifest.elements[i]);
@ -432,7 +435,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
attrPath, attrPath,
}; };
pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME
} }
} }

View file

@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon
auto app = installable->toApp(*state); auto app = installable->toApp(*state);
state->store->buildPaths(app.context); state->store->buildPaths(toBuildableReqs(app.context));
Strings allArgs{app.program}; Strings allArgs{app.program};
for (auto & i : args) allArgs.push_back(i); for (auto & i : args) allArgs.push_back(i);