Allow remote builds without sending the derivation closure

Previously, to build a derivation remotely, we had to copy the entire
closure of the .drv file to the remote machine, even though we only
need the top-level derivation. This is very wasteful: the closure can
contain thousands of store paths, and in some Hydra use cases, include
source paths that are very large (e.g. Git/Mercurial checkouts).

So now there is a new operation, StoreAPI::buildDerivation(), that
performs a build from an in-memory representation of a derivation
(BasicDerivation) rather than from a on-disk .drv file. The only files
that need to be in the Nix store are the sources of the derivation
(drv.inputSrcs), and the needed output paths of the dependencies (as
described by drv.inputDrvs). "nix-store --serve" exposes this
interface.

Note that this is a privileged operation, because you can construct a
derivation that builds any store path whatsoever. Fixing this will
require changing the hashing scheme (i.e., the output paths should be
computed from the other fields in BasicDerivation, allowing them to be
verified without access to other derivations). However, this would be
quite nice because it would allow .drv-free building (e.g. "nix-env
-i" wouldn't have to write any .drv files to disk).

Fixes #173.
This commit is contained in:
Eelco Dolstra 2015-07-17 17:57:40 +02:00
parent f39979c6d3
commit 1511aa9f48
12 changed files with 372 additions and 154 deletions

View file

@ -275,6 +275,7 @@ public:
/* Make a goal (with caching). */ /* Make a goal (with caching). */
GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
GoalPtr makeBasicDerivationGoal(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal);
GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
/* Remove a dead goal. */ /* Remove a dead goal. */
@ -690,6 +691,9 @@ class SubstitutionGoal;
class DerivationGoal : public Goal class DerivationGoal : public Goal
{ {
private: private:
/* Whether to use an on-disk .drv file. */
bool useDerivation;
/* The path of the derivation. */ /* The path of the derivation. */
Path drvPath; Path drvPath;
@ -698,14 +702,14 @@ private:
StringSet wantedOutputs; StringSet wantedOutputs;
/* Whether additional wanted outputs have been added. */ /* Whether additional wanted outputs have been added. */
bool needRestart; bool needRestart = false;
/* Whether to retry substituting the outputs after building the /* Whether to retry substituting the outputs after building the
inputs. */ inputs. */
bool retrySubstitution; bool retrySubstitution = false;
/* The derivation stored at drvPath. */ /* The derivation stored at drvPath. */
Derivation drv; std::unique_ptr<BasicDerivation> drv;
/* The remainder is state held during the build. */ /* The remainder is state held during the build. */
@ -736,8 +740,8 @@ private:
Path tmpDir; Path tmpDir;
/* File descriptor for the log file. */ /* File descriptor for the log file. */
FILE * fLogFile; FILE * fLogFile = 0;
BZFILE * bzLogFile; BZFILE * bzLogFile = 0;
AutoCloseFD fdLogFile; AutoCloseFD fdLogFile;
/* Number of bytes received from the builder's stdout/stderr. */ /* Number of bytes received from the builder's stdout/stderr. */
@ -750,7 +754,7 @@ private:
std::shared_ptr<HookInstance> hook; std::shared_ptr<HookInstance> hook;
/* Whether we're currently doing a chroot build. */ /* Whether we're currently doing a chroot build. */
bool useChroot; bool useChroot = false;
Path chrootRootDir; Path chrootRootDir;
@ -790,7 +794,10 @@ private:
InodesSeen inodesSeen; InodesSeen inodesSeen;
public: public:
DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
Worker & worker, BuildMode buildMode = bmNormal);
DerivationGoal(const Path & drvPath, const BasicDerivation & drv,
Worker & worker, BuildMode buildMode = bmNormal);
~DerivationGoal(); ~DerivationGoal();
void cancel(bool timeout); void cancel(bool timeout);
@ -816,7 +823,8 @@ public:
private: private:
/* The states. */ /* The states. */
void init(); void getDerivation();
void loadDerivation();
void haveDerivation(); void haveDerivation();
void outputsSubstituted(); void outputsSubstituted();
void closureRepaired(); void closureRepaired();
@ -867,23 +875,34 @@ private:
}; };
DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
Worker & worker, BuildMode buildMode)
: Goal(worker) : Goal(worker)
, useDerivation(true)
, drvPath(drvPath)
, wantedOutputs(wantedOutputs) , wantedOutputs(wantedOutputs)
, needRestart(false)
, retrySubstitution(false)
, fLogFile(0)
, bzLogFile(0)
, useChroot(false)
, buildMode(buildMode) , buildMode(buildMode)
{ {
this->drvPath = drvPath; state = &DerivationGoal::getDerivation;
state = &DerivationGoal::init;
name = (format("building of %1%") % drvPath).str(); name = (format("building of %1%") % drvPath).str();
trace("created"); trace("created");
} }
DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv,
Worker & worker, BuildMode buildMode)
: Goal(worker)
, useDerivation(false)
, drvPath(drvPath)
, buildMode(buildMode)
{
this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv));
state = &DerivationGoal::haveDerivation;
name = (format("building of %1%") % showPaths(outputPaths(drv))).str();
trace("created");
}
DerivationGoal::~DerivationGoal() DerivationGoal::~DerivationGoal()
{ {
/* Careful: we should never ever throw an exception from a /* Careful: we should never ever throw an exception from a
@ -951,28 +970,25 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs)
} }
void DerivationGoal::init() void DerivationGoal::getDerivation()
{ {
trace("init"); trace("init");
if (settings.readOnlyMode)
throw Error(format("cannot build derivation %1% - no write access to the Nix store") % drvPath);
/* The first thing to do is to make sure that the derivation /* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a exists. If it doesn't, it may be created through a
substitute. */ substitute. */
if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
haveDerivation(); loadDerivation();
return; return;
} }
addWaitee(worker.makeSubstitutionGoal(drvPath)); addWaitee(worker.makeSubstitutionGoal(drvPath));
state = &DerivationGoal::haveDerivation; state = &DerivationGoal::loadDerivation;
} }
void DerivationGoal::haveDerivation() void DerivationGoal::loadDerivation()
{ {
trace("loading derivation"); trace("loading derivation");
@ -990,10 +1006,18 @@ void DerivationGoal::haveDerivation()
assert(worker.store.isValidPath(drvPath)); assert(worker.store.isValidPath(drvPath));
/* Get the derivation. */ /* Get the derivation. */
drv = derivationFromPath(worker.store, drvPath); drv = std::unique_ptr<BasicDerivation>(new Derivation(derivationFromPath(worker.store, drvPath)));
foreach (DerivationOutputs::iterator, i, drv.outputs) haveDerivation();
worker.store.addTempRoot(i->second.path); }
void DerivationGoal::haveDerivation()
{
trace("have derivation");
for (auto & i : drv->outputs)
worker.store.addTempRoot(i.second.path);
/* Check what outputs paths are not already valid. */ /* Check what outputs paths are not already valid. */
PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@ -1006,15 +1030,15 @@ void DerivationGoal::haveDerivation()
/* Check whether any output previously failed to build. If so, /* Check whether any output previously failed to build. If so,
don't bother. */ don't bother. */
foreach (PathSet::iterator, i, invalidOutputs) for (auto & i : invalidOutputs)
if (pathFailed(*i)) return; if (pathFailed(i)) return;
/* We are first going to try to create the invalid output paths /* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build through substitutes. If that doesn't work, we'll build
them. */ them. */
if (settings.useSubstitutes && substitutesAllowed(drv)) if (settings.useSubstitutes && substitutesAllowed(*drv))
foreach (PathSet::iterator, i, invalidOutputs) for (auto & i : invalidOutputs)
addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair)); addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair));
if (waitees.empty()) /* to prevent hang (no wake-up event) */ if (waitees.empty()) /* to prevent hang (no wake-up event) */
outputsSubstituted(); outputsSubstituted();
@ -1063,11 +1087,17 @@ void DerivationGoal::outputsSubstituted()
wantedOutputs = PathSet(); wantedOutputs = PathSet();
/* The inputs must be built before we can build this goal. */ /* The inputs must be built before we can build this goal. */
foreach (DerivationInputs::iterator, i, drv.inputDrvs) if (useDerivation)
addWaitee(worker.makeDerivationGoal(i->first, i->second, buildMode == bmRepair ? bmRepair : bmNormal)); for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
foreach (PathSet::iterator, i, drv.inputSrcs) for (auto & i : drv->inputSrcs) {
addWaitee(worker.makeSubstitutionGoal(*i)); if (worker.store.isValidPath(i)) continue;
if (!settings.useSubstitutes)
throw Error(format("dependency of %1% of %2% does not exist, and substitution is disabled")
% i % drvPath);
addWaitee(worker.makeSubstitutionGoal(i));
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */ if (waitees.empty()) /* to prevent hang (no wake-up event) */
inputsRealised(); inputsRealised();
@ -1085,34 +1115,34 @@ void DerivationGoal::repairClosure()
/* Get the output closure. */ /* Get the output closure. */
PathSet outputClosure; PathSet outputClosure;
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
computeFSClosure(worker.store, i->second.path, outputClosure); computeFSClosure(worker.store, i.second.path, outputClosure);
/* Filter out our own outputs (which we have already checked). */ /* Filter out our own outputs (which we have already checked). */
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
outputClosure.erase(i->second.path); outputClosure.erase(i.second.path);
/* Get all dependencies of this derivation so that we know which /* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output derivation is responsible for which path in the output
closure. */ closure. */
PathSet inputClosure; PathSet inputClosure;
computeFSClosure(worker.store, drvPath, inputClosure); if (useDerivation) computeFSClosure(worker.store, drvPath, inputClosure);
std::map<Path, Path> outputsToDrv; std::map<Path, Path> outputsToDrv;
foreach (PathSet::iterator, i, inputClosure) for (auto & i : inputClosure)
if (isDerivation(*i)) { if (isDerivation(i)) {
Derivation drv = derivationFromPath(worker.store, *i); Derivation drv = derivationFromPath(worker.store, i);
foreach (DerivationOutputs::iterator, j, drv.outputs) for (auto & j : drv.outputs)
outputsToDrv[j->second.path] = *i; outputsToDrv[j.second.path] = i;
} }
/* Check each path (slow!). */ /* Check each path (slow!). */
PathSet broken; PathSet broken;
foreach (PathSet::iterator, i, outputClosure) { for (auto & i : outputClosure) {
if (worker.store.pathContentsGood(*i)) continue; if (worker.store.pathContentsGood(i)) continue;
printMsg(lvlError, format("found corrupted or missing path %1% in the output closure of %2%") % *i % drvPath); printMsg(lvlError, format("found corrupted or missing path %1% in the output closure of %2%") % i % drvPath);
Path drvPath2 = outputsToDrv[*i]; Path drvPath2 = outputsToDrv[i];
if (drvPath2 == "") if (drvPath2 == "")
addWaitee(worker.makeSubstitutionGoal(*i, true)); addWaitee(worker.makeSubstitutionGoal(i, true));
else else
addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair));
} }
@ -1140,6 +1170,8 @@ void DerivationGoal::inputsRealised()
trace("all inputs realised"); trace("all inputs realised");
if (nrFailed != 0) { if (nrFailed != 0) {
if (!useDerivation)
throw Error(format("some dependencies of %1% are missing") % drvPath);
printMsg(lvlError, printMsg(lvlError,
format("cannot build derivation %1%: %2% dependencies couldn't be built") format("cannot build derivation %1%: %2% dependencies couldn't be built")
% drvPath % nrFailed); % drvPath % nrFailed);
@ -1156,32 +1188,33 @@ void DerivationGoal::inputsRealised()
running the build hook. */ running the build hook. */
/* The outputs are referenceable paths. */ /* The outputs are referenceable paths. */
foreach (DerivationOutputs::iterator, i, drv.outputs) { for (auto & i : drv->outputs) {
debug(format("building path %1%") % i->second.path); debug(format("building path %1%") % i.second.path);
allPaths.insert(i->second.path); allPaths.insert(i.second.path);
} }
/* Determine the full set of input paths. */ /* Determine the full set of input paths. */
/* First, the input derivations. */ /* First, the input derivations. */
foreach (DerivationInputs::iterator, i, drv.inputDrvs) { if (useDerivation)
/* Add the relevant output closures of the input derivation for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
`*i' as input paths. Only add the closures of output paths /* Add the relevant output closures of the input derivation
that are specified as inputs. */ `i' as input paths. Only add the closures of output paths
assert(worker.store.isValidPath(i->first)); that are specified as inputs. */
Derivation inDrv = derivationFromPath(worker.store, i->first); assert(worker.store.isValidPath(i.first));
foreach (StringSet::iterator, j, i->second) Derivation inDrv = derivationFromPath(worker.store, i.first);
if (inDrv.outputs.find(*j) != inDrv.outputs.end()) for (auto & j : i.second)
computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths); if (inDrv.outputs.find(j) != inDrv.outputs.end())
else computeFSClosure(worker.store, inDrv.outputs[j].path, inputPaths);
throw Error( else
format("derivation %1% requires non-existent output %2% from input derivation %3%") throw Error(
% drvPath % *j % i->first); format("derivation %1% requires non-existent output %2% from input derivation %3%")
} % drvPath % j % i.first);
}
/* Second, the input sources. */ /* Second, the input sources. */
foreach (PathSet::iterator, i, drv.inputSrcs) for (auto & i : drv->inputSrcs)
computeFSClosure(worker.store, *i, inputPaths); computeFSClosure(worker.store, i, inputPaths);
debug(format("added input paths %1%") % showPaths(inputPaths)); debug(format("added input paths %1%") % showPaths(inputPaths));
@ -1189,8 +1222,8 @@ void DerivationGoal::inputsRealised()
/* Is this a fixed-output derivation? */ /* Is this a fixed-output derivation? */
fixedOutput = true; fixedOutput = true;
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
if (i->second.hash == "") fixedOutput = false; if (i.second.hash == "") fixedOutput = false;
/* Okay, try to build. Note that here we don't wait for a build /* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a slot to become available, since we don't need one if there is a
@ -1217,13 +1250,13 @@ static string get(const StringPairs & map, const string & key, const string & de
} }
bool willBuildLocally(const Derivation & drv) bool willBuildLocally(const BasicDerivation & drv)
{ {
return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform);
} }
bool substitutesAllowed(const Derivation & drv) bool substitutesAllowed(const BasicDerivation & drv)
{ {
return get(drv.env, "allowSubstitutes", "1") == "1"; return get(drv.env, "allowSubstitutes", "1") == "1";
} }
@ -1238,10 +1271,10 @@ void DerivationGoal::tryToBuild()
(It can't happen between here and the lockPaths() call below (It can't happen between here and the lockPaths() call below
because we're not allowing multi-threading.) If so, put this because we're not allowing multi-threading.) If so, put this
goal to sleep until another goal finishes, then try again. */ goal to sleep until another goal finishes, then try again. */
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
if (pathIsLockedByMe(i->second.path)) { if (pathIsLockedByMe(i.second.path)) {
debug(format("putting derivation %1% to sleep because %2% is locked by another goal") debug(format("putting derivation %1% to sleep because %2% is locked by another goal")
% drvPath % i->second.path); % drvPath % i.second.path);
worker.waitForAnyGoal(shared_from_this()); worker.waitForAnyGoal(shared_from_this());
return; return;
} }
@ -1251,7 +1284,7 @@ void DerivationGoal::tryToBuild()
can't acquire the lock, then continue; hopefully some other can't acquire the lock, then continue; hopefully some other
goal can start a build, and if not, the main loop will sleep a goal can start a build, and if not, the main loop will sleep a
few seconds and then retry this goal. */ few seconds and then retry this goal. */
if (!outputLocks.lockPaths(outputPaths(drv), "", false)) { if (!outputLocks.lockPaths(outputPaths(*drv), "", false)) {
worker.waitForAWhile(shared_from_this()); worker.waitForAWhile(shared_from_this());
return; return;
} }
@ -1264,22 +1297,22 @@ void DerivationGoal::tryToBuild()
now hold the locks on the output paths, no other process can now hold the locks on the output paths, no other process can
build this derivation, so no further checks are necessary. */ build this derivation, so no further checks are necessary. */
validPaths = checkPathValidity(true, buildMode == bmRepair); validPaths = checkPathValidity(true, buildMode == bmRepair);
assert(buildMode != bmCheck || validPaths.size() == drv.outputs.size()); assert(buildMode != bmCheck || validPaths.size() == drv->outputs.size());
if (buildMode != bmCheck && validPaths.size() == drv.outputs.size()) { if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) {
debug(format("skipping build of derivation %1%, someone beat us to it") % drvPath); debug(format("skipping build of derivation %1%, someone beat us to it") % drvPath);
outputLocks.setDeletion(true); outputLocks.setDeletion(true);
amDone(ecSuccess); amDone(ecSuccess);
return; return;
} }
missingPaths = outputPaths(drv); missingPaths = outputPaths(*drv);
if (buildMode != bmCheck) if (buildMode != bmCheck)
foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i); foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i);
/* If any of the outputs already exist but are not valid, delete /* If any of the outputs already exist but are not valid, delete
them. */ them. */
foreach (DerivationOutputs::iterator, i, drv.outputs) { for (auto & i : drv->outputs) {
Path path = i->second.path; Path path = i.second.path;
if (worker.store.isValidPath(path)) continue; if (worker.store.isValidPath(path)) continue;
if (!pathExists(path)) continue; if (!pathExists(path)) continue;
debug(format("removing invalid path %1%") % path); debug(format("removing invalid path %1%") % path);
@ -1289,13 +1322,13 @@ void DerivationGoal::tryToBuild()
/* Check again whether any output previously failed to build, /* Check again whether any output previously failed to build,
because some other process may have tried and failed before we because some other process may have tried and failed before we
acquired the lock. */ acquired the lock. */
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
if (pathFailed(i->second.path)) return; if (pathFailed(i.second.path)) return;
/* Don't do a remote build if the derivation has the attribute /* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only `preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */ supported for local builds. */
bool buildLocally = buildMode != bmNormal || willBuildLocally(drv); bool buildLocally = buildMode != bmNormal || willBuildLocally(*drv);
/* Is the build hook willing to accept this job? */ /* Is the build hook willing to accept this job? */
if (!buildLocally) { if (!buildLocally) {
@ -1504,8 +1537,8 @@ void DerivationGoal::buildDone()
Hook errors (like communication problems with the Hook errors (like communication problems with the
remote machine) shouldn't be cached either. */ remote machine) shouldn't be cached either. */
if (settings.cacheFailure && !fixedOutput && !diskFull) if (settings.cacheFailure && !fixedOutput && !diskFull)
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
worker.store.registerFailedPath(i->second.path); worker.store.registerFailedPath(i.second.path);
} }
amDone(ecFailed); amDone(ecFailed);
@ -1524,21 +1557,21 @@ void DerivationGoal::buildDone()
HookReply DerivationGoal::tryBuildHook() HookReply DerivationGoal::tryBuildHook()
{ {
if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline;
if (!worker.hook) if (!worker.hook)
worker.hook = std::shared_ptr<HookInstance>(new HookInstance); worker.hook = std::make_shared<HookInstance>();
/* Tell the hook about system features (beyond the system type) /* Tell the hook about system features (beyond the system type)
required from the build machine. (The hook could parse the required from the build machine. (The hook could parse the
drv file itself, but this is easier.) */ drv file itself, but this is easier.) */
Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures")); Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures"));
foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */
/* Send the request to the hook. */ /* Send the request to the hook. */
writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
% (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
% drv.platform % drvPath % concatStringsSep(",", features)).str()); % drv->platform % drvPath % concatStringsSep(",", features)).str());
/* Read the first line of input, which should be a word indicating /* Read the first line of input, which should be a word indicating
whether the hook wishes to perform the build. */ whether the hook wishes to perform the build. */
@ -1596,7 +1629,7 @@ HookReply DerivationGoal::tryBuildHook()
if (settings.printBuildTrace) if (settings.printBuildTrace)
printMsg(lvlError, format("@ build-started %1% - %2% %3%") printMsg(lvlError, format("@ build-started %1% - %2% %3%")
% drvPath % drv.platform % logFile); % drvPath % drv->platform % logFile);
return rpAccept; return rpAccept;
} }
@ -1624,12 +1657,12 @@ void DerivationGoal::startBuilder()
"building path(s) %1%") % showPaths(missingPaths)); "building path(s) %1%") % showPaths(missingPaths));
/* Right platform? */ /* Right platform? */
if (!canBuildLocally(drv.platform)) { if (!canBuildLocally(drv->platform)) {
if (settings.printBuildTrace) if (settings.printBuildTrace)
printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform); printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform);
throw Error( throw Error(
format("a %1% is required to build %3%, but I am a %2%") format("a %1% is required to build %3%, but I am a %2%")
% drv.platform % settings.thisSystem % drvPath); % drv->platform % settings.thisSystem % drvPath);
} }
/* Construct the environment passed to the builder. */ /* Construct the environment passed to the builder. */
@ -1666,9 +1699,9 @@ void DerivationGoal::startBuilder()
attribute. Those are passed as file names pointing to attribute. Those are passed as file names pointing to
temporary files containing the contents. */ temporary files containing the contents. */
PathSet filesToChown; PathSet filesToChown;
StringSet passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile")); StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
int fileNr = 0; int fileNr = 0;
for (auto & i : drv.env) { for (auto & i : drv->env) {
if (passAsFile.find(i.first) == passAsFile.end()) { if (passAsFile.find(i.first) == passAsFile.end()) {
env[i.first] = i.second; env[i.first] = i.second;
} else { } else {
@ -1708,7 +1741,7 @@ void DerivationGoal::startBuilder()
fixed-output derivations is by definition pure (since we fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */ already know the cryptographic hash of the output). */
if (fixedOutput) { if (fixedOutput) {
Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars")); Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars"));
foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i); foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i);
} }
@ -1719,7 +1752,7 @@ void DerivationGoal::startBuilder()
temporary build directory. The text files have the format used temporary build directory. The text files have the format used
by `nix-store --register-validity'. However, the deriver by `nix-store --register-validity'. However, the deriver
fields are left empty. */ fields are left empty. */
string s = get(drv.env, "exportReferencesGraph"); string s = get(drv->env, "exportReferencesGraph");
Strings ss = tokenizeString<Strings>(s); Strings ss = tokenizeString<Strings>(s);
if (ss.size() % 2 != 0) if (ss.size() % 2 != 0)
throw BuildError(format("odd number of tokens in exportReferencesGraph: %1%") % s); throw BuildError(format("odd number of tokens in exportReferencesGraph: %1%") % s);
@ -1748,8 +1781,8 @@ void DerivationGoal::startBuilder()
foreach (PathSet::iterator, j, paths2) { foreach (PathSet::iterator, j, paths2) {
if (isDerivation(*j)) { if (isDerivation(*j)) {
Derivation drv = derivationFromPath(worker.store, *j); Derivation drv = derivationFromPath(worker.store, *j);
foreach (DerivationOutputs::iterator, k, drv.outputs) for (auto & k : drv.outputs)
computeFSClosure(worker.store, k->second.path, paths); computeFSClosure(worker.store, k.second.path, paths);
} }
} }
@ -1789,14 +1822,14 @@ void DerivationGoal::startBuilder()
if (x != "true" && x != "false" && x != "relaxed") if (x != "true" && x != "false" && x != "relaxed")
throw Error("option build-use-chroot must be set to one of true, false or relaxed"); throw Error("option build-use-chroot must be set to one of true, false or relaxed");
if (x == "true") { if (x == "true") {
if (get(drv.env, "__noChroot") == "1") if (get(drv->env, "__noChroot") == "1")
throw Error(format("derivation %1% has __noChroot set, but that's not allowed when build-use-chroot is true") % drvPath); throw Error(format("derivation %1% has __noChroot set, but that's not allowed when build-use-chroot is true") % drvPath);
useChroot = true; useChroot = true;
} }
else if (x == "false") else if (x == "false")
useChroot = false; useChroot = false;
else if (x == "relaxed") else if (x == "relaxed")
useChroot = !fixedOutput && get(drv.env, "__noChroot") != "1"; useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
} }
if (useChroot) { if (useChroot) {
@ -1834,7 +1867,7 @@ void DerivationGoal::startBuilder()
PathSet allowedPaths = tokenizeString<StringSet>(allowed); PathSet allowedPaths = tokenizeString<StringSet>(allowed);
/* This works like the above, except on a per-derivation level */ /* This works like the above, except on a per-derivation level */
Strings impurePaths = tokenizeString<Strings>(get(drv.env, "__impureHostDeps")); Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps"));
for (auto & i : impurePaths) { for (auto & i : impurePaths) {
bool found = false; bool found = false;
@ -1865,7 +1898,7 @@ void DerivationGoal::startBuilder()
if (pathExists(chrootRootDir)) deletePath(chrootRootDir); if (pathExists(chrootRootDir)) deletePath(chrootRootDir);
/* Clean up the chroot directory automatically. */ /* Clean up the chroot directory automatically. */
autoDelChroot = std::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
printMsg(lvlChatty, format("setting up chroot environment in %1%") % chrootRootDir); printMsg(lvlChatty, format("setting up chroot environment in %1%") % chrootRootDir);
@ -1948,8 +1981,8 @@ void DerivationGoal::startBuilder()
(typically the dependencies of /bin/sh). Throw them (typically the dependencies of /bin/sh). Throw them
out. */ out. */
if (buildMode != bmNormal) if (buildMode != bmNormal)
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
dirsInChroot.erase(i->second.path); dirsInChroot.erase(i.second.path);
#elif SANDBOX_ENABLED #elif SANDBOX_ENABLED
/* We don't really have any parent prep work to do (yet?) /* We don't really have any parent prep work to do (yet?)
@ -2025,7 +2058,7 @@ void DerivationGoal::startBuilder()
} }
/* Run the builder. */ /* Run the builder. */
printMsg(lvlChatty, format("executing builder %1%") % drv.builder); printMsg(lvlChatty, format("executing builder %1%") % drv->builder);
/* Create the log file. */ /* Create the log file. */
Path logFile = openLogFile(); Path logFile = openLogFile();
@ -2111,7 +2144,7 @@ void DerivationGoal::startBuilder()
if (settings.printBuildTrace) { if (settings.printBuildTrace) {
printMsg(lvlError, format("@ build-started %1% - %2% %3%") printMsg(lvlError, format("@ build-started %1% - %2% %3%")
% drvPath % drv.platform % logFile); % drvPath % drv->platform % logFile);
} }
} }
@ -2285,7 +2318,7 @@ void DerivationGoal::runChild()
i686-linux build on an x86_64-linux machine. */ i686-linux build on an x86_64-linux machine. */
struct utsname utsbuf; struct utsname utsbuf;
uname(&utsbuf); uname(&utsbuf);
if (drv.platform == "i686-linux" && if (drv->platform == "i686-linux" &&
(settings.thisSystem == "x86_64-linux" || (settings.thisSystem == "x86_64-linux" ||
(!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
if (personality(PER_LINUX32) == -1) if (personality(PER_LINUX32) == -1)
@ -2294,7 +2327,7 @@ void DerivationGoal::runChild()
/* Impersonate a Linux 2.6 machine to get some determinism in /* Impersonate a Linux 2.6 machine to get some determinism in
builds that depend on the kernel version. */ builds that depend on the kernel version. */
if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) { if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
int cur = personality(0xffffffff); int cur = personality(0xffffffff);
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
} }
@ -2449,15 +2482,15 @@ void DerivationGoal::runChild()
args.push_back("sandbox-exec"); args.push_back("sandbox-exec");
args.push_back("-p"); args.push_back("-p");
args.push_back(sandboxProfile); args.push_back(sandboxProfile);
args.push_back(drv.builder); args.push_back(drv->builder);
} else { } else {
builder = drv.builder.c_str(); builder = drv->builder.c_str();
string builderBasename = baseNameOf(drv.builder); string builderBasename = baseNameOf(drv->builder);
args.push_back(builderBasename); args.push_back(builderBasename);
} }
foreach (Strings::iterator, i, drv.args) for (auto & i : drv->args)
args.push_back(rewriteHashes(*i, rewritesToTmp)); args.push_back(rewriteHashes(i, rewritesToTmp));
restoreSIGPIPE(); restoreSIGPIPE();
@ -2473,7 +2506,7 @@ void DerivationGoal::runChild()
/* Execute the program. This should not return. */ /* Execute the program. This should not return. */
execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
throw SysError(format("executing %1%") % drv.builder); throw SysError(format("executing %1%") % drv->builder);
} catch (std::exception & e) { } catch (std::exception & e) {
writeFull(STDERR_FILENO, "while setting up the build environment: " + string(e.what()) + "\n"); writeFull(STDERR_FILENO, "while setting up the build environment: " + string(e.what()) + "\n");
@ -2485,7 +2518,7 @@ void DerivationGoal::runChild()
/* Parse a list of reference specifiers. Each element must either be /* Parse a list of reference specifiers. Each element must either be
a store path, or the symbolic name of the output of the derivation a store path, or the symbolic name of the output of the derivation
(such as `out'). */ (such as `out'). */
PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) PathSet parseReferenceSpecifiers(const BasicDerivation & drv, string attr)
{ {
PathSet result; PathSet result;
Paths paths = tokenizeString<Paths>(attr); Paths paths = tokenizeString<Paths>(attr);
@ -2509,8 +2542,8 @@ void DerivationGoal::registerOutputs()
to do anything here. */ to do anything here. */
if (hook) { if (hook) {
bool allValid = true; bool allValid = true;
foreach (DerivationOutputs::iterator, i, drv.outputs) for (auto & i : drv->outputs)
if (!worker.store.isValidPath(i->second.path)) allValid = false; if (!worker.store.isValidPath(i.second.path)) allValid = false;
if (allValid) return; if (allValid) return;
} }
@ -2519,8 +2552,8 @@ void DerivationGoal::registerOutputs()
/* Check whether the output paths were created, and grep each /* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all output path to determine what other paths it references. Also make all
output paths read-only. */ output paths read-only. */
foreach (DerivationOutputs::iterator, i, drv.outputs) { for (auto & i : drv->outputs) {
Path path = i->second.path; Path path = i.second.path;
if (missingPaths.find(path) == missingPaths.end()) continue; if (missingPaths.find(path) == missingPaths.end()) continue;
Path actualPath = path; Path actualPath = path;
@ -2591,10 +2624,10 @@ void DerivationGoal::registerOutputs()
/* Check that fixed-output derivations produced the right /* Check that fixed-output derivations produced the right
outputs (i.e., the content hash should match the specified outputs (i.e., the content hash should match the specified
hash). */ hash). */
if (i->second.hash != "") { if (i.second.hash != "") {
bool recursive; HashType ht; Hash h; bool recursive; HashType ht; Hash h;
i->second.parseHashInfo(recursive, ht, h); i.second.parseHashInfo(recursive, ht, h);
if (!recursive) { if (!recursive) {
/* The output path should be a regular file without /* The output path should be a regular file without
@ -2609,7 +2642,7 @@ void DerivationGoal::registerOutputs()
if (h != h2) if (h != h2)
throw BuildError( throw BuildError(
format("output path %1% should have %2% hash %3%, instead has %4%") format("output path %1% should have %2% hash %3%, instead has %4%")
% path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); % path % i.second.hashAlgo % printHash16or32(h) % printHash16or32(h2));
} }
/* Get rid of all weird permissions. This also checks that /* Get rid of all weird permissions. This also checks that
@ -2643,9 +2676,9 @@ void DerivationGoal::registerOutputs()
/* Enforce `allowedReferences' and friends. */ /* Enforce `allowedReferences' and friends. */
auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
if (drv.env.find(attrName) == drv.env.end()) return; if (drv->env.find(attrName) == drv->env.end()) return;
PathSet spec = parseReferenceSpecifiers(drv, get(drv.env, attrName)); PathSet spec = parseReferenceSpecifiers(*drv, get(drv->env, attrName));
PathSet used; PathSet used;
if (recursive) { if (recursive) {
@ -2806,12 +2839,12 @@ void DerivationGoal::handleEOF(int fd)
PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
{ {
PathSet result; PathSet result;
foreach (DerivationOutputs::iterator, i, drv.outputs) { for (auto & i : drv->outputs) {
if (!wantOutput(i->first, wantedOutputs)) continue; if (!wantOutput(i.first, wantedOutputs)) continue;
bool good = bool good =
worker.store.isValidPath(i->second.path) && worker.store.isValidPath(i.second.path) &&
(!checkHash || worker.store.pathContentsGood(i->second.path)); (!checkHash || worker.store.pathContentsGood(i.second.path));
if (good == returnValid) result.insert(i->second.path); if (good == returnValid) result.insert(i.second.path);
} }
return result; return result;
} }
@ -3066,7 +3099,7 @@ void SubstitutionGoal::tryToRun()
} }
/* Acquire a lock on the output path. */ /* Acquire a lock on the output path. */
outputLock = std::shared_ptr<PathLocks>(new PathLocks); outputLock = std::make_shared<PathLocks>();
if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) {
worker.waitForAWhile(shared_from_this()); worker.waitForAWhile(shared_from_this());
return; return;
@ -3270,7 +3303,7 @@ GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOu
{ {
GoalPtr goal = derivationGoals[path].lock(); GoalPtr goal = derivationGoals[path].lock();
if (!goal) { if (!goal) {
goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, buildMode)); goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode);
derivationGoals[path] = goal; derivationGoals[path] = goal;
wakeUp(goal); wakeUp(goal);
} else } else
@ -3279,11 +3312,19 @@ GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOu
} }
GoalPtr Worker::makeBasicDerivationGoal(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode)
{
auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode);
wakeUp(goal);
return goal;
}
GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair)
{ {
GoalPtr goal = substitutionGoals[path].lock(); GoalPtr goal = substitutionGoals[path].lock();
if (!goal) { if (!goal) {
goal = GoalPtr(new SubstitutionGoal(path, *this, repair)); goal = std::make_shared<SubstitutionGoal>(path, *this, repair);
substitutionGoals[path] = goal; substitutionGoals[path] = goal;
wakeUp(goal); wakeUp(goal);
} }
@ -3604,8 +3645,7 @@ unsigned int Worker::exitStatus()
void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
{ {
startNest(nest, lvlDebug, startNest(nest, lvlDebug, format("building %1%") % showPaths(drvPaths));
format("building %1%") % showPaths(drvPaths));
Worker worker(*this); Worker worker(*this);
@ -3633,6 +3673,35 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
} }
BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
startNest(nest, lvlDebug, format("building %1%") % showPaths({drvPath}));
Worker worker(*this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode);
BuildResult result;
try {
worker.run(Goals{goal});
if (goal->getExitCode() == Goal::ecSuccess)
result.status = BuildResult::Success;
else if (worker.permanentFailure)
result.status = BuildResult::PermanentFailure;
else if (worker.timedOut)
result.status = BuildResult::TimedOut;
else
result.status = BuildResult::MiscFailure;
} catch (Error & e) {
result.status = BuildResult::MiscFailure;
result.errorMsg = e.msg();
}
return result;
}
void LocalStore::ensurePath(const Path & path) void LocalStore::ensurePath(const Path & path)
{ {
/* If the path is already valid, we're done. */ /* If the path is already valid, we're done. */

View file

@ -3,6 +3,7 @@
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include "util.hh"
#include "misc.hh" #include "misc.hh"
#include "worker-protocol.hh"
namespace nix { namespace nix {
@ -285,7 +286,7 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
} }
PathSet outputPaths(const Derivation & drv) PathSet outputPaths(const BasicDerivation & drv)
{ {
PathSet paths; PathSet paths;
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
@ -294,4 +295,44 @@ PathSet outputPaths(const Derivation & drv)
} }
Source & operator >> (Source & in, BasicDerivation & drv)
{
drv.outputs.clear();
auto nr = readInt(in);
for (unsigned int n = 0; n < nr; n++) {
auto name = readString(in);
DerivationOutput o;
in >> o.path >> o.hashAlgo >> o.hash;
assertStorePath(o.path);
drv.outputs[name] = o;
}
drv.inputSrcs = readStorePaths<PathSet>(in);
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);
nr = readInt(in);
for (unsigned int n = 0; n < nr; n++) {
auto key = readString(in);
auto value = readString(in);
drv.env[key] = value;
}
return in;
}
Sink & operator << (Sink & out, const BasicDerivation & drv)
{
out << drv.outputs.size();
for (auto & i : drv.outputs)
out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)
out << i.first << i.second;
return out;
}
} }

View file

@ -40,15 +40,21 @@ typedef std::map<Path, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs; typedef std::map<string, string> StringPairs;
struct Derivation struct BasicDerivation
{ {
DerivationOutputs outputs; /* keyed on symbolic IDs */ DerivationOutputs outputs; /* keyed on symbolic IDs */
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
PathSet inputSrcs; /* inputs that are sources */ PathSet inputSrcs; /* inputs that are sources */
string platform; string platform;
Path builder; Path builder;
Strings args; Strings args;
StringPairs env; StringPairs env;
virtual ~BasicDerivation() { };
};
struct Derivation : BasicDerivation
{
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
}; };
@ -89,6 +95,12 @@ Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outpu
bool wantOutput(const string & output, const std::set<string> & wanted); bool wantOutput(const string & output, const std::set<string> & wanted);
PathSet outputPaths(const Derivation & drv); PathSet outputPaths(const BasicDerivation & drv);
struct Source;
struct Sink;
Source & operator >> (Source & in, BasicDerivation & drv);
Sink & operator << (Sink & out, const BasicDerivation & drv);
} }

View file

@ -151,6 +151,9 @@ public:
void buildPaths(const PathSet & paths, BuildMode buildMode); void buildPaths(const PathSet & paths, BuildMode buildMode);
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
void ensurePath(const Path & path); void ensurePath(const Path & path);
void addTempRoot(const Path & path); void addTempRoot(const Path & path);

View file

@ -32,9 +32,9 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize); unsigned long long & downloadSize, unsigned long long & narSize);
bool willBuildLocally(const Derivation & drv); bool willBuildLocally(const BasicDerivation & drv);
bool substitutesAllowed(const Derivation & drv); bool substitutesAllowed(const BasicDerivation & drv);
} }

View file

@ -482,6 +482,13 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
} }
BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
throw Error("not implemented");
}
void RemoteStore::ensurePath(const Path & path) void RemoteStore::ensurePath(const Path & path)
{ {
openConnection(); openConnection();

View file

@ -67,6 +67,9 @@ public:
void buildPaths(const PathSet & paths, BuildMode buildMode); void buildPaths(const PathSet & paths, BuildMode buildMode);
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
void ensurePath(const Path & path); void ensurePath(const Path & path);
void addTempRoot(const Path & path); void addTempRoot(const Path & path);

View file

@ -100,6 +100,22 @@ typedef list<ValidPathInfo> ValidPathInfos;
enum BuildMode { bmNormal, bmRepair, bmCheck }; enum BuildMode { bmNormal, bmRepair, bmCheck };
struct BuildResult
{
enum Status {
Success = 0,
PermanentFailure = 1,
TimedOut = 2,
MiscFailure = 3
} status = MiscFailure;
std::string errorMsg;
//time_t startTime = 0, stopTime = 0;
};
struct BasicDerivation;
class StoreAPI class StoreAPI
{ {
public: public:
@ -194,6 +210,12 @@ public:
not derivations, substitute them. */ not derivations, substitute them. */
virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0;
/* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file). Note that drvPath is only used for
informational purposes. */
virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode = bmNormal) = 0;
/* Ensure that a path is valid. If it is not currently valid, it /* Ensure that a path is valid. If it is not currently valid, it
may be made valid by running a substitute (if defined for the may be made valid by running a substitute (if defined for the
path). */ path). */

View file

@ -155,6 +155,12 @@ void writeInt(unsigned int n, Sink & sink)
sink(buf, sizeof(buf)); sink(buf, sizeof(buf));
} }
Sink & operator << (Sink & out, unsigned int n)
{
writeInt(n, out);
return out;
}
void writeLongLong(unsigned long long n, Sink & sink) void writeLongLong(unsigned long long n, Sink & sink)
{ {
@ -184,6 +190,12 @@ void writeString(const string & s, Sink & sink)
writeString((const unsigned char *) s.data(), s.size(), sink); writeString((const unsigned char *) s.data(), s.size(), sink);
} }
Sink & operator << (Sink & out, const string & s)
{
writeString(s, out);
return out;
}
template<class T> void writeStrings(const T & ss, Sink & sink) template<class T> void writeStrings(const T & ss, Sink & sink)
{ {
@ -195,6 +207,18 @@ template<class T> void writeStrings(const T & ss, Sink & sink)
template void writeStrings(const Paths & ss, Sink & sink); template void writeStrings(const Paths & ss, Sink & sink);
template void writeStrings(const PathSet & ss, Sink & sink); template void writeStrings(const PathSet & ss, Sink & sink);
Sink & operator << (Sink & out, const Strings & s)
{
writeStrings(s, out);
return out;
}
Sink & operator << (Sink & out, const StringSet & s)
{
writeStrings(s, out);
return out;
}
void readPadding(size_t len, Source & source) void readPadding(size_t len, Source & source)
{ {
@ -258,6 +282,12 @@ string readString(Source & source)
return string((char *) buf, len); return string((char *) buf, len);
} }
Source & operator >> (Source & in, string & s)
{
s = readString(in);
return in;
}
template<class T> T readStrings(Source & source) template<class T> T readStrings(Source & source)
{ {

View file

@ -118,6 +118,12 @@ void writeString(const unsigned char * buf, size_t len, Sink & sink);
void writeString(const string & s, Sink & sink); void writeString(const string & s, Sink & sink);
template<class T> void writeStrings(const T & ss, Sink & sink); template<class T> void writeStrings(const T & ss, Sink & sink);
Sink & operator << (Sink & out, unsigned int n);
Sink & operator << (Sink & out, const string & s);
Sink & operator << (Sink & out, const Strings & s);
Sink & operator << (Sink & out, const StringSet & s);
void readPadding(size_t len, Source & source); void readPadding(size_t len, Source & source);
unsigned int readInt(Source & source); unsigned int readInt(Source & source);
unsigned long long readLongLong(Source & source); unsigned long long readLongLong(Source & source);
@ -125,6 +131,8 @@ size_t readString(unsigned char * buf, size_t max, Source & source);
string readString(Source & source); string readString(Source & source);
template<class T> T readStrings(Source & source); template<class T> T readStrings(Source & source);
Source & operator >> (Source & in, string & s);
MakeError(SerialisationError, Error) MakeError(SerialisationError, Error)

View file

@ -862,6 +862,16 @@ static void opServe(Strings opFlags, Strings opArgs)
out.flush(); out.flush();
readInt(in); // Client version, unused for now readInt(in); // Client version, unused for now
auto getBuildSettings = [&]() {
// FIXME: changing options here doesn't work if we're
// building through the daemon.
verbosity = lvlError;
settings.keepLog = false;
settings.useSubstitutes = false;
settings.maxSilentTime = readInt(in);
settings.buildTimeout = readInt(in);
};
while (true) { while (true) {
ServeCommand cmd; ServeCommand cmd;
try { try {
@ -943,19 +953,12 @@ static void opServe(Strings opFlags, Strings opArgs)
break; break;
} }
case cmdBuildPaths: { case cmdBuildPaths: { /* Used by build-remote.pl. */
/* Used by build-remote.pl. */
if (!writeAllowed) throw Error("building paths is not allowed"); if (!writeAllowed) throw Error("building paths is not allowed");
PathSet paths = readStorePaths<PathSet>(in); PathSet paths = readStorePaths<PathSet>(in);
// FIXME: changing options here doesn't work if we're getBuildSettings();
// building through the daemon.
verbosity = lvlError;
settings.keepLog = false;
settings.useSubstitutes = false;
settings.maxSilentTime = readInt(in);
settings.buildTimeout = readInt(in);
try { try {
MonitorFdHup monitor(in.fd); MonitorFdHup monitor(in.fd);
@ -969,6 +972,25 @@ static void opServe(Strings opFlags, Strings opArgs)
break; break;
} }
case cmdBuildDerivation: { /* Used by hydra-queue-runner. */
if (!writeAllowed) throw Error("building paths is not allowed");
Path drvPath = readStorePath(in); // informational only
BasicDerivation drv;
in >> drv;
getBuildSettings();
MonitorFdHup monitor(in.fd);
auto status = store->buildDerivation(drvPath, drv);
writeInt(status.status, out);
writeString(status.errorMsg, out);
break;
}
case cmdQueryClosure: { case cmdQueryClosure: {
bool includeOutputs = readInt(in); bool includeOutputs = readInt(in);
PathSet paths = readStorePaths<PathSet>(in); PathSet paths = readStorePaths<PathSet>(in);

View file

@ -5,7 +5,7 @@ namespace nix {
#define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_1 0x390c9deb
#define SERVE_MAGIC_2 0x5452eecb #define SERVE_MAGIC_2 0x5452eecb
#define SERVE_PROTOCOL_VERSION 0x200 #define SERVE_PROTOCOL_VERSION 0x201
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -17,6 +17,7 @@ typedef enum {
cmdExportPaths = 5, cmdExportPaths = 5,
cmdBuildPaths = 6, cmdBuildPaths = 6,
cmdQueryClosure = 7, cmdQueryClosure = 7,
cmdBuildDerivation = 8,
} ServeCommand; } ServeCommand;
} }