mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-22 05:56:15 +02:00
* A flag `--keep-going / -k' to keep building goals if one fails, as
much as possible. (This is similar to GNU Make's `-k' flag.) * Refactoring to implement this: previously we just bombed out when a build failed, but now we have to clean up. In particular this means that goals must be freed quickly --- they shouldn't hang around until the worker exits. So the worker now maintains weak pointers in order not to prevent garbage collection. * Documented the `-k' and `-j' flags.
This commit is contained in:
parent
e4883211f9
commit
b113edeab7
10 changed files with 209 additions and 144 deletions
|
@ -4,5 +4,14 @@
|
||||||
<arg rep='repeat'><option>-v</option></arg>
|
<arg rep='repeat'><option>-v</option></arg>
|
||||||
<arg><option>--build-output</option></arg>
|
<arg><option>--build-output</option></arg>
|
||||||
<arg><option>-B</option></arg>
|
<arg><option>-B</option></arg>
|
||||||
|
<arg>
|
||||||
|
<group choice='req'>
|
||||||
|
<arg choice='plain'><option>--max-jobs</option></arg>
|
||||||
|
<arg choice='plain'><option>-j</option></arg>
|
||||||
|
</group>
|
||||||
|
<replaceable>number</replaceable>
|
||||||
|
</arg>
|
||||||
|
<arg><option>--keep-going</option></arg>
|
||||||
|
<arg><option>-k</option></arg>
|
||||||
<arg><option>--keep-failed</option></arg>
|
<arg><option>--keep-failed</option></arg>
|
||||||
<arg><option>-K</option></arg>
|
<arg><option>-K</option></arg>
|
||||||
|
|
|
@ -108,6 +108,33 @@
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
|
<varlistentry>
|
||||||
|
<term><option>--max-jobs</option> / <option>-j</option></term>
|
||||||
|
<listitem>
|
||||||
|
<para>
|
||||||
|
Sets the maximum number of build jobs that Nix will perform in
|
||||||
|
parallel to the specified number. The default is 1. A higher
|
||||||
|
value is useful on SMP systems or to exploit I/O latency.
|
||||||
|
</para>
|
||||||
|
</listitem>
|
||||||
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
|
<varlistentry>
|
||||||
|
<term><option>--keep-going</option> / <option>-k</option></term>
|
||||||
|
<listitem>
|
||||||
|
<para>
|
||||||
|
Keep going in case of failed builds, to the greatest extent
|
||||||
|
possible. That is, if building an input of some derivation
|
||||||
|
fails, Nix will still build the other inputs, but not the
|
||||||
|
derivation itself. Without this option, Nix stops if any build
|
||||||
|
fails (except for builds of substitutes), possibly killing
|
||||||
|
builds in progress (in case of parallel or distributed builds).
|
||||||
|
</para>
|
||||||
|
</listitem>
|
||||||
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
<varlistentry>
|
<varlistentry>
|
||||||
<term><option>--keep-failed</option> / <option>-K</option></term>
|
<term><option>--keep-failed</option> / <option>-K</option></term>
|
||||||
<listitem>
|
<listitem>
|
||||||
|
|
|
@ -106,3 +106,4 @@ The hook `nix-mode-hook' is run when Nix mode is started.
|
||||||
|
|
||||||
|
|
||||||
(setq auto-mode-alist (cons '("\\.nix\\'" . nix-mode) auto-mode-alist))
|
(setq auto-mode-alist (cons '("\\.nix\\'" . nix-mode) auto-mode-alist))
|
||||||
|
(setq auto-mode-alist (cons '("\\.nix.in\\'" . nix-mode) auto-mode-alist))
|
||||||
|
|
|
@ -137,6 +137,8 @@ static void initAndRun(int argc, char * * argv)
|
||||||
}
|
}
|
||||||
else if (arg == "--keep-failed" || arg == "-K")
|
else if (arg == "--keep-failed" || arg == "-K")
|
||||||
keepFailed = true;
|
keepFailed = true;
|
||||||
|
else if (arg == "--keep-going" || arg == "-k")
|
||||||
|
keepGoing = true;
|
||||||
else if (arg == "--max-jobs" || arg == "-j") {
|
else if (arg == "--max-jobs" || arg == "-j") {
|
||||||
++i;
|
++i;
|
||||||
if (i == args.end()) throw UsageError("`--max-jobs' requires an argument");
|
if (i == args.end()) throw UsageError("`--max-jobs' requires an argument");
|
||||||
|
|
|
@ -8,6 +8,8 @@ string nixDBPath = "/UNINIT";
|
||||||
|
|
||||||
bool keepFailed = false;
|
bool keepFailed = false;
|
||||||
|
|
||||||
|
bool keepGoing = false;
|
||||||
|
|
||||||
Verbosity buildVerbosity = lvlDebug;
|
Verbosity buildVerbosity = lvlDebug;
|
||||||
|
|
||||||
unsigned int maxBuildJobs = 1;
|
unsigned int maxBuildJobs = 1;
|
||||||
|
|
|
@ -29,6 +29,10 @@ extern string nixDBPath;
|
||||||
/* Whether to keep temporary directories of failed builds. */
|
/* Whether to keep temporary directories of failed builds. */
|
||||||
extern bool keepFailed;
|
extern bool keepFailed;
|
||||||
|
|
||||||
|
/* Whether to keep building subgoals when a sibling (another subgoal
|
||||||
|
of the same goal) fails. */
|
||||||
|
extern bool keepGoing;
|
||||||
|
|
||||||
/* Verbosity level for build output. */
|
/* Verbosity level for build output. */
|
||||||
extern Verbosity buildVerbosity;
|
extern Verbosity buildVerbosity;
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <boost/shared_ptr.hpp>
|
#include <boost/shared_ptr.hpp>
|
||||||
|
#include <boost/weak_ptr.hpp>
|
||||||
#include <boost/enable_shared_from_this.hpp>
|
#include <boost/enable_shared_from_this.hpp>
|
||||||
|
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
@ -26,13 +27,14 @@ class Worker;
|
||||||
/* A pointer to a goal. */
|
/* A pointer to a goal. */
|
||||||
class Goal;
|
class Goal;
|
||||||
typedef shared_ptr<Goal> GoalPtr;
|
typedef shared_ptr<Goal> GoalPtr;
|
||||||
|
typedef weak_ptr<Goal> WeakGoalPtr;
|
||||||
|
|
||||||
/* A set of goals. */
|
/* Set of goals. */
|
||||||
typedef set<GoalPtr> Goals;
|
typedef set<GoalPtr> Goals;
|
||||||
|
typedef set<WeakGoalPtr> WeakGoals;
|
||||||
|
|
||||||
/* A map of paths to goals (and the other way around). */
|
/* A map of paths to goals (and the other way around). */
|
||||||
typedef map<Path, GoalPtr> GoalMap;
|
typedef map<Path, WeakGoalPtr> WeakGoalMap;
|
||||||
typedef map<GoalPtr, Path> GoalMapRev;
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -43,8 +45,12 @@ protected:
|
||||||
/* Backlink to the worker. */
|
/* Backlink to the worker. */
|
||||||
Worker & worker;
|
Worker & worker;
|
||||||
|
|
||||||
/* Goals waiting for this one to finish. */
|
/* Goals that this goal is waiting for. */
|
||||||
Goals waiters;
|
Goals waitees;
|
||||||
|
|
||||||
|
/* Goals waiting for this one to finish. Must use weak pointers
|
||||||
|
here to prevent cycles. */
|
||||||
|
WeakGoals waiters;
|
||||||
|
|
||||||
/* Number of goals we are waiting for. */
|
/* Number of goals we are waiting for. */
|
||||||
unsigned int nrWaitees;
|
unsigned int nrWaitees;
|
||||||
|
@ -75,20 +81,15 @@ protected:
|
||||||
public:
|
public:
|
||||||
virtual void work() = 0;
|
virtual void work() = 0;
|
||||||
|
|
||||||
virtual string name()
|
virtual string name() = 0;
|
||||||
{
|
|
||||||
return "(noname)";
|
|
||||||
}
|
|
||||||
|
|
||||||
void addWaiter(GoalPtr waiter);
|
void addWaitee(GoalPtr waitee);
|
||||||
|
|
||||||
virtual void waiteeDone(bool success);
|
virtual void waiteeDone(GoalPtr waitee, bool success);
|
||||||
|
|
||||||
|
void trace(const format & f);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
virtual void waiteeFailed()
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void amDone(bool success = true);
|
void amDone(bool success = true);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -97,7 +98,7 @@ protected:
|
||||||
belongs, and a file descriptor for receiving log data. */
|
belongs, and a file descriptor for receiving log data. */
|
||||||
struct Child
|
struct Child
|
||||||
{
|
{
|
||||||
GoalPtr goal;
|
WeakGoalPtr goal;
|
||||||
int fdOutput;
|
int fdOutput;
|
||||||
bool inBuildSlot;
|
bool inBuildSlot;
|
||||||
};
|
};
|
||||||
|
@ -110,14 +111,17 @@ class Worker
|
||||||
{
|
{
|
||||||
private:
|
private:
|
||||||
|
|
||||||
/* The goals of the worker. */
|
/* Note: the worker should only have strong pointers to the
|
||||||
Goals goals;
|
top-level goals. */
|
||||||
|
|
||||||
|
/* The top-level goals of the worker. */
|
||||||
|
Goals topGoals;
|
||||||
|
|
||||||
/* Goals that are ready to do some work. */
|
/* Goals that are ready to do some work. */
|
||||||
Goals awake;
|
WeakGoals awake;
|
||||||
|
|
||||||
/* Goals waiting for a build slot. */
|
/* Goals waiting for a build slot. */
|
||||||
Goals wantingToBuild;
|
WeakGoals wantingToBuild;
|
||||||
|
|
||||||
/* Child processes currently running. */
|
/* Child processes currently running. */
|
||||||
Children children;
|
Children children;
|
||||||
|
@ -128,24 +132,21 @@ private:
|
||||||
|
|
||||||
/* Maps used to prevent multiple instantiation of a goal for the
|
/* Maps used to prevent multiple instantiation of a goal for the
|
||||||
same expression / path. */
|
same expression / path. */
|
||||||
GoalMap normalisationGoals;
|
WeakGoalMap normalisationGoals;
|
||||||
GoalMapRev normalisationGoalsRev;
|
WeakGoalMap realisationGoals;
|
||||||
GoalMap realisationGoals;
|
WeakGoalMap substitutionGoals;
|
||||||
GoalMapRev realisationGoalsRev;
|
|
||||||
GoalMap substitutionGoals;
|
|
||||||
GoalMapRev substitutionGoalsRev;
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
Worker();
|
Worker();
|
||||||
~Worker();
|
~Worker();
|
||||||
|
|
||||||
/* Add a goal. */
|
/* Make a goal (with caching). */
|
||||||
void addNormalisationGoal(const Path & nePath, GoalPtr waiter);
|
GoalPtr makeNormalisationGoal(const Path & nePath);
|
||||||
void addRealisationGoal(const Path & nePath, GoalPtr waiter);
|
GoalPtr makeRealisationGoal(const Path & nePath);
|
||||||
void addSubstitutionGoal(const Path & storePath, GoalPtr waiter);
|
GoalPtr makeSubstitutionGoal(const Path & storePath);
|
||||||
|
|
||||||
/* Remove a finished goal. */
|
/* Remove a dead goal. */
|
||||||
void removeGoal(GoalPtr goal);
|
void removeGoal(GoalPtr goal);
|
||||||
|
|
||||||
/* Wake up a goal (i.e., there is something for it to do). */
|
/* Wake up a goal (i.e., there is something for it to do). */
|
||||||
|
@ -162,8 +163,9 @@ public:
|
||||||
/* Add a goal to the set of goals waiting for a build slot. */
|
/* Add a goal to the set of goals waiting for a build slot. */
|
||||||
void waitForBuildSlot(GoalPtr goal);
|
void waitForBuildSlot(GoalPtr goal);
|
||||||
|
|
||||||
/* Loop until all goals have been realised. */
|
/* Loop until the specified top-level goal has finished. Returns
|
||||||
void run();
|
true if it has finished succesfully. */
|
||||||
|
bool run(GoalPtr topGoal);
|
||||||
|
|
||||||
/* Wait for input to become available. */
|
/* Wait for input to become available. */
|
||||||
void waitForInput();
|
void waitForInput();
|
||||||
|
@ -188,35 +190,44 @@ public:
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
|
||||||
void Goal::addWaiter(GoalPtr waiter)
|
void Goal::addWaitee(GoalPtr waitee)
|
||||||
{
|
{
|
||||||
waiters.insert(waiter);
|
waitees.insert(waitee);
|
||||||
|
waitee->waiters.insert(shared_from_this());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Goal::waiteeDone(bool success)
|
void Goal::waiteeDone(GoalPtr waitee, bool success)
|
||||||
{
|
{
|
||||||
|
assert(waitees.find(waitee) != waitees.end());
|
||||||
|
waitees.erase(waitee);
|
||||||
assert(nrWaitees > 0);
|
assert(nrWaitees > 0);
|
||||||
/* Note: waiteeFailed should never call amDone()! */
|
if (!success) ++nrFailed;
|
||||||
if (!success) {
|
if (!--nrWaitees || (!success && !keepGoing))
|
||||||
++nrFailed;
|
worker.wakeUp(shared_from_this());
|
||||||
waiteeFailed();
|
|
||||||
}
|
|
||||||
if (!--nrWaitees) worker.wakeUp(shared_from_this());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Goal::amDone(bool success)
|
void Goal::amDone(bool success)
|
||||||
{
|
{
|
||||||
printMsg(lvlVomit, "done");
|
trace("done");
|
||||||
assert(!done);
|
assert(!done);
|
||||||
done = true;
|
done = true;
|
||||||
for (Goals::iterator i = waiters.begin(); i != waiters.end(); ++i)
|
for (WeakGoals::iterator i = waiters.begin(); i != waiters.end(); ++i) {
|
||||||
(*i)->waiteeDone(success);
|
GoalPtr goal = i->lock();
|
||||||
|
if (goal) goal->waiteeDone(shared_from_this(), success);
|
||||||
|
}
|
||||||
|
waiters.clear();
|
||||||
worker.removeGoal(shared_from_this());
|
worker.removeGoal(shared_from_this());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void Goal::trace(const format & f)
|
||||||
|
{
|
||||||
|
debug(format("%1%: %2%") % name() % f);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
@ -356,12 +367,7 @@ private:
|
||||||
/* Delete the temporary directory, if we have one. */
|
/* Delete the temporary directory, if we have one. */
|
||||||
void deleteTmpDir(bool force);
|
void deleteTmpDir(bool force);
|
||||||
|
|
||||||
string name()
|
string name();
|
||||||
{
|
|
||||||
return nePath;
|
|
||||||
}
|
|
||||||
|
|
||||||
void trace(const format & f);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -375,6 +381,8 @@ NormalisationGoal::NormalisationGoal(const Path & _nePath, Worker & _worker)
|
||||||
|
|
||||||
NormalisationGoal::~NormalisationGoal()
|
NormalisationGoal::~NormalisationGoal()
|
||||||
{
|
{
|
||||||
|
if (pid != -1) worker.childTerminated(pid);
|
||||||
|
|
||||||
/* Careful: we should never ever throw an exception from a
|
/* Careful: we should never ever throw an exception from a
|
||||||
destructor. */
|
destructor. */
|
||||||
try {
|
try {
|
||||||
|
@ -407,7 +415,7 @@ void NormalisationGoal::init()
|
||||||
exists. If it doesn't, it may be created through a
|
exists. If it doesn't, it may be created through a
|
||||||
substitute. */
|
substitute. */
|
||||||
resetWaitees(1);
|
resetWaitees(1);
|
||||||
worker.addSubstitutionGoal(nePath, shared_from_this());
|
addWaitee(worker.makeSubstitutionGoal(nePath));
|
||||||
|
|
||||||
state = &NormalisationGoal::haveStoreExpr;
|
state = &NormalisationGoal::haveStoreExpr;
|
||||||
}
|
}
|
||||||
|
@ -440,7 +448,7 @@ void NormalisationGoal::haveStoreExpr()
|
||||||
/* Inputs must be normalised before we can build this goal. */
|
/* Inputs must be normalised before we can build this goal. */
|
||||||
for (PathSet::iterator i = expr.derivation.inputs.begin();
|
for (PathSet::iterator i = expr.derivation.inputs.begin();
|
||||||
i != expr.derivation.inputs.end(); ++i)
|
i != expr.derivation.inputs.end(); ++i)
|
||||||
worker.addNormalisationGoal(*i, shared_from_this());
|
addWaitee(worker.makeNormalisationGoal(*i));
|
||||||
|
|
||||||
resetWaitees(expr.derivation.inputs.size());
|
resetWaitees(expr.derivation.inputs.size());
|
||||||
|
|
||||||
|
@ -469,7 +477,7 @@ void NormalisationGoal::inputNormalised()
|
||||||
if (querySuccessor(neInput, nfInput))
|
if (querySuccessor(neInput, nfInput))
|
||||||
neInput = nfInput;
|
neInput = nfInput;
|
||||||
/* Otherwise the input must be a closure. */
|
/* Otherwise the input must be a closure. */
|
||||||
worker.addRealisationGoal(neInput, shared_from_this());
|
addWaitee(worker.makeRealisationGoal(neInput));
|
||||||
}
|
}
|
||||||
|
|
||||||
resetWaitees(expr.derivation.inputs.size());
|
resetWaitees(expr.derivation.inputs.size());
|
||||||
|
@ -1153,9 +1161,9 @@ void NormalisationGoal::deleteTmpDir(bool force)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void NormalisationGoal::trace(const format & f)
|
string NormalisationGoal::name()
|
||||||
{
|
{
|
||||||
debug(format("normalisation of `%1%': %2%") % nePath % f);
|
return (format("normalisation of `%1%'") % nePath).str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -1186,7 +1194,7 @@ public:
|
||||||
void haveStoreExpr();
|
void haveStoreExpr();
|
||||||
void elemFinished();
|
void elemFinished();
|
||||||
|
|
||||||
void trace(const format & f);
|
string name();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -1217,7 +1225,7 @@ void RealisationGoal::init()
|
||||||
exists. If it doesn't, it may be created through a
|
exists. If it doesn't, it may be created through a
|
||||||
substitute. */
|
substitute. */
|
||||||
resetWaitees(1);
|
resetWaitees(1);
|
||||||
worker.addSubstitutionGoal(nePath, shared_from_this());
|
addWaitee(worker.makeSubstitutionGoal(nePath));
|
||||||
|
|
||||||
state = &RealisationGoal::haveStoreExpr;
|
state = &RealisationGoal::haveStoreExpr;
|
||||||
}
|
}
|
||||||
|
@ -1248,7 +1256,7 @@ void RealisationGoal::haveStoreExpr()
|
||||||
through a substitute. */
|
through a substitute. */
|
||||||
for (ClosureElems::const_iterator i = expr.closure.elems.begin();
|
for (ClosureElems::const_iterator i = expr.closure.elems.begin();
|
||||||
i != expr.closure.elems.end(); ++i)
|
i != expr.closure.elems.end(); ++i)
|
||||||
worker.addSubstitutionGoal(i->first, shared_from_this());
|
addWaitee(worker.makeSubstitutionGoal(i->first));
|
||||||
|
|
||||||
resetWaitees(expr.closure.elems.size());
|
resetWaitees(expr.closure.elems.size());
|
||||||
|
|
||||||
|
@ -1274,9 +1282,9 @@ void RealisationGoal::elemFinished()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void RealisationGoal::trace(const format & f)
|
string RealisationGoal::name()
|
||||||
{
|
{
|
||||||
debug(format("realisation of `%1%': %2%") % nePath % f);
|
return (format("realisation of `%1%'") % nePath).str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -1313,6 +1321,7 @@ private:
|
||||||
|
|
||||||
public:
|
public:
|
||||||
SubstitutionGoal(const Path & _nePath, Worker & _worker);
|
SubstitutionGoal(const Path & _nePath, Worker & _worker);
|
||||||
|
~SubstitutionGoal();
|
||||||
|
|
||||||
void work();
|
void work();
|
||||||
|
|
||||||
|
@ -1324,7 +1333,7 @@ public:
|
||||||
void tryToRun();
|
void tryToRun();
|
||||||
void finished();
|
void finished();
|
||||||
|
|
||||||
void trace(const format & f);
|
string name();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -1336,6 +1345,12 @@ SubstitutionGoal::SubstitutionGoal(const Path & _storePath, Worker & _worker)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
SubstitutionGoal::~SubstitutionGoal()
|
||||||
|
{
|
||||||
|
if (pid != -1) worker.childTerminated(pid);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void SubstitutionGoal::work()
|
void SubstitutionGoal::work()
|
||||||
{
|
{
|
||||||
(this->*state)();
|
(this->*state)();
|
||||||
|
@ -1377,7 +1392,7 @@ void SubstitutionGoal::tryNext()
|
||||||
subs.pop_front();
|
subs.pop_front();
|
||||||
|
|
||||||
/* Normalise the substitute store expression. */
|
/* Normalise the substitute store expression. */
|
||||||
worker.addNormalisationGoal(sub.storeExpr, shared_from_this());
|
addWaitee(worker.makeNormalisationGoal(sub.storeExpr));
|
||||||
resetWaitees(1);
|
resetWaitees(1);
|
||||||
|
|
||||||
state = &SubstitutionGoal::exprNormalised;
|
state = &SubstitutionGoal::exprNormalised;
|
||||||
|
@ -1396,7 +1411,7 @@ void SubstitutionGoal::exprNormalised()
|
||||||
/* Realise the substitute store expression. */
|
/* Realise the substitute store expression. */
|
||||||
if (!querySuccessor(sub.storeExpr, nfSub))
|
if (!querySuccessor(sub.storeExpr, nfSub))
|
||||||
nfSub = sub.storeExpr;
|
nfSub = sub.storeExpr;
|
||||||
worker.addRealisationGoal(nfSub, shared_from_this());
|
addWaitee(worker.makeRealisationGoal(nfSub));
|
||||||
|
|
||||||
resetWaitees(1);
|
resetWaitees(1);
|
||||||
|
|
||||||
|
@ -1557,9 +1572,9 @@ void SubstitutionGoal::finished()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void SubstitutionGoal::trace(const format & f)
|
string SubstitutionGoal::name()
|
||||||
{
|
{
|
||||||
debug(format("substitution of `%1%': %2%") % storePath % f);
|
return (format("substitution of `%1%'") % storePath).str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -1585,7 +1600,7 @@ public:
|
||||||
abort();
|
abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
void waiteeDone(bool success)
|
void waiteeDone(GoalPtr waitee, bool success)
|
||||||
{
|
{
|
||||||
if (!success) this->success = false;
|
if (!success) this->success = false;
|
||||||
}
|
}
|
||||||
|
@ -1594,6 +1609,11 @@ public:
|
||||||
{
|
{
|
||||||
return success;
|
return success;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string name()
|
||||||
|
{
|
||||||
|
return "pseudo-goal";
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -1616,71 +1636,67 @@ Worker::Worker()
|
||||||
Worker::~Worker()
|
Worker::~Worker()
|
||||||
{
|
{
|
||||||
working = false;
|
working = false;
|
||||||
|
|
||||||
|
/* Explicitly get rid of all strong pointers now. After this all
|
||||||
|
goals that refer to this worker should be gone. (Otherwise we
|
||||||
|
are in trouble, since goals may call childTerminated() etc. in
|
||||||
|
their destructors). */
|
||||||
|
topGoals.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
template<class T>
|
template<class T>
|
||||||
static void addGoal(const Path & path, GoalPtr waiter,
|
static GoalPtr addGoal(const Path & path,
|
||||||
Worker & worker, Goals & goals,
|
Worker & worker, WeakGoalMap & goalMap)
|
||||||
GoalMap & goalMap, GoalMapRev & goalMapRev)
|
|
||||||
{
|
{
|
||||||
GoalPtr goal;
|
GoalPtr goal = goalMap[path].lock();
|
||||||
goal = goalMap[path];
|
|
||||||
if (!goal) {
|
if (!goal) {
|
||||||
goal = GoalPtr(new T(path, worker));
|
goal = GoalPtr(new T(path, worker));
|
||||||
goals.insert(goal);
|
|
||||||
goalMap[path] = goal;
|
goalMap[path] = goal;
|
||||||
goalMapRev[goal] = path;
|
|
||||||
worker.wakeUp(goal);
|
worker.wakeUp(goal);
|
||||||
}
|
}
|
||||||
if (waiter) goal->addWaiter(waiter);
|
return goal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Worker::addNormalisationGoal(const Path & nePath, GoalPtr waiter)
|
GoalPtr Worker::makeNormalisationGoal(const Path & nePath)
|
||||||
{
|
{
|
||||||
addGoal<NormalisationGoal>(nePath, waiter, *this, goals,
|
return addGoal<NormalisationGoal>(nePath, *this, normalisationGoals);
|
||||||
normalisationGoals, normalisationGoalsRev);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Worker::addRealisationGoal(const Path & nePath, GoalPtr waiter)
|
GoalPtr Worker::makeRealisationGoal(const Path & nePath)
|
||||||
{
|
{
|
||||||
addGoal<RealisationGoal>(nePath, waiter, *this, goals,
|
return addGoal<RealisationGoal>(nePath, *this, realisationGoals);
|
||||||
realisationGoals, realisationGoalsRev);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Worker::addSubstitutionGoal(const Path & storePath, GoalPtr waiter)
|
GoalPtr Worker::makeSubstitutionGoal(const Path & storePath)
|
||||||
{
|
{
|
||||||
addGoal<SubstitutionGoal>(storePath, waiter, *this, goals,
|
return addGoal<SubstitutionGoal>(storePath, *this, substitutionGoals);
|
||||||
substitutionGoals, substitutionGoalsRev);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void removeGoal(GoalPtr goal,
|
static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
|
||||||
GoalMap & goalMap, GoalMapRev & goalMapRev)
|
|
||||||
{
|
{
|
||||||
GoalMapRev::iterator i = goalMapRev.find(goal);
|
/* !!! For now we just let dead goals accumulate. We should
|
||||||
if (i != goalMapRev.end()) {
|
probably periodically sweep the goalMap to remove dead
|
||||||
goalMapRev.erase(i);
|
goals. */
|
||||||
goalMap.erase(i->second);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Worker::removeGoal(GoalPtr goal)
|
void Worker::removeGoal(GoalPtr goal)
|
||||||
{
|
{
|
||||||
goals.erase(goal);
|
topGoals.erase(goal);
|
||||||
::removeGoal(goal, normalisationGoals, normalisationGoalsRev);
|
::removeGoal(goal, normalisationGoals);
|
||||||
::removeGoal(goal, realisationGoals, realisationGoalsRev);
|
::removeGoal(goal, realisationGoals);
|
||||||
::removeGoal(goal, substitutionGoals, substitutionGoalsRev);
|
::removeGoal(goal, substitutionGoals);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Worker::wakeUp(GoalPtr goal)
|
void Worker::wakeUp(GoalPtr goal)
|
||||||
{
|
{
|
||||||
printMsg(lvlVomit, "wake up");
|
goal->trace("woken up");
|
||||||
awake.insert(goal);
|
awake.insert(goal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1716,9 +1732,12 @@ void Worker::childTerminated(pid_t pid)
|
||||||
children.erase(pid);
|
children.erase(pid);
|
||||||
|
|
||||||
/* Wake up goals waiting for a build slot. */
|
/* Wake up goals waiting for a build slot. */
|
||||||
for (Goals::iterator i = wantingToBuild.begin();
|
for (WeakGoals::iterator i = wantingToBuild.begin();
|
||||||
i != wantingToBuild.end(); ++i)
|
i != wantingToBuild.end(); ++i)
|
||||||
wakeUp(*i);
|
{
|
||||||
|
GoalPtr goal = i->lock();
|
||||||
|
if (goal) wakeUp(goal);
|
||||||
|
}
|
||||||
wantingToBuild.clear();
|
wantingToBuild.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1733,8 +1752,18 @@ void Worker::waitForBuildSlot(GoalPtr goal)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Worker::run()
|
bool Worker::run(GoalPtr topGoal)
|
||||||
{
|
{
|
||||||
|
assert(topGoal);
|
||||||
|
|
||||||
|
/* Wrap the specified top-level goal in a pseudo-goal so that we
|
||||||
|
can check whether it succeeded. */
|
||||||
|
shared_ptr<PseudoGoal> pseudo(new PseudoGoal(*this));
|
||||||
|
pseudo->addWaitee(topGoal);
|
||||||
|
|
||||||
|
/* For now, we have only one top-level goal. */
|
||||||
|
topGoals.insert(topGoal);
|
||||||
|
|
||||||
startNest(nest, lvlDebug, format("entered goal loop"));
|
startNest(nest, lvlDebug, format("entered goal loop"));
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
|
@ -1743,18 +1772,16 @@ void Worker::run()
|
||||||
|
|
||||||
/* Call every wake goal. */
|
/* Call every wake goal. */
|
||||||
while (!awake.empty()) {
|
while (!awake.empty()) {
|
||||||
Goals awake2(awake); /* !!! why is this necessary? */
|
WeakGoals awake2(awake);
|
||||||
awake.clear();
|
awake.clear();
|
||||||
for (Goals::iterator i = awake2.begin(); i != awake2.end(); ++i) {
|
for (WeakGoals::iterator i = awake2.begin(); i != awake2.end(); ++i) {
|
||||||
printMsg(lvlVomit,
|
|
||||||
format("running goal (%1% left)") % goals.size());
|
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
GoalPtr goal = *i;
|
GoalPtr goal = i->lock();
|
||||||
goal->work();
|
if (goal) goal->work();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (goals.empty()) break;
|
if (topGoals.empty()) break;
|
||||||
|
|
||||||
/* !!! not when we're polling */
|
/* !!! not when we're polling */
|
||||||
assert(!children.empty());
|
assert(!children.empty());
|
||||||
|
@ -1763,9 +1790,14 @@ void Worker::run()
|
||||||
waitForInput();
|
waitForInput();
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(awake.empty());
|
/* If --keep-going is not set, it's possible that the main goal
|
||||||
assert(wantingToBuild.empty());
|
exited while some of its subgoals were still active. But if
|
||||||
assert(children.empty());
|
--keep-going *is* set, then they must all be finished now. */
|
||||||
|
assert(!keepGoing || awake.empty());
|
||||||
|
assert(!keepGoing || wantingToBuild.empty());
|
||||||
|
assert(!keepGoing || children.empty());
|
||||||
|
|
||||||
|
return pseudo->isOkay();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -1801,22 +1833,23 @@ void Worker::waitForInput()
|
||||||
i != children.end(); ++i)
|
i != children.end(); ++i)
|
||||||
{
|
{
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
GoalPtr goal = i->second.goal;
|
GoalPtr goal = i->second.goal.lock();
|
||||||
|
assert(goal);
|
||||||
int fd = i->second.fdOutput;
|
int fd = i->second.fdOutput;
|
||||||
if (FD_ISSET(fd, &fds)) {
|
if (FD_ISSET(fd, &fds)) {
|
||||||
unsigned char buffer[4096];
|
unsigned char buffer[4096];
|
||||||
ssize_t rd = read(fd, buffer, sizeof(buffer));
|
ssize_t rd = read(fd, buffer, sizeof(buffer));
|
||||||
if (rd == -1) {
|
if (rd == -1) {
|
||||||
if (errno != EINTR)
|
if (errno != EINTR)
|
||||||
throw SysError(format("reading from `%1%'")
|
throw SysError(format("reading from %1%")
|
||||||
% goal->name());
|
% goal->name());
|
||||||
} else if (rd == 0) {
|
} else if (rd == 0) {
|
||||||
debug(format("EOF on `%1%'") % goal->name());
|
debug(format("%1%: got EOF") % goal->name());
|
||||||
wakeUp(goal);
|
wakeUp(goal);
|
||||||
} else {
|
} else {
|
||||||
printMsg(lvlVomit, format("read %1% bytes from `%2%'")
|
printMsg(lvlVomit, format("%1%: read %2% bytes")
|
||||||
% rd % goal->name());
|
% goal->name() % rd);
|
||||||
// writeFull(goal.fdLogFile, buffer, rd);
|
// writeFull(goal.fdLogFile, buffer, rd); !!!
|
||||||
if (verbosity >= buildVerbosity)
|
if (verbosity >= buildVerbosity)
|
||||||
writeFull(STDERR_FILENO, buffer, rd);
|
writeFull(STDERR_FILENO, buffer, rd);
|
||||||
}
|
}
|
||||||
|
@ -1833,11 +1866,7 @@ Path normaliseStoreExpr(const Path & nePath)
|
||||||
startNest(nest, lvlDebug, format("normalising `%1%'") % nePath);
|
startNest(nest, lvlDebug, format("normalising `%1%'") % nePath);
|
||||||
|
|
||||||
Worker worker;
|
Worker worker;
|
||||||
shared_ptr<PseudoGoal> pseudo(new PseudoGoal(worker));
|
if (!worker.run(worker.makeNormalisationGoal(nePath)))
|
||||||
worker.addNormalisationGoal(nePath, pseudo);
|
|
||||||
worker.run();
|
|
||||||
|
|
||||||
if (!pseudo->isOkay())
|
|
||||||
throw Error(format("normalisation of store expression `%1%' failed") % nePath);
|
throw Error(format("normalisation of store expression `%1%' failed") % nePath);
|
||||||
|
|
||||||
Path nfPath;
|
Path nfPath;
|
||||||
|
@ -1851,11 +1880,7 @@ void realiseClosure(const Path & nePath)
|
||||||
startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath);
|
startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath);
|
||||||
|
|
||||||
Worker worker;
|
Worker worker;
|
||||||
shared_ptr<PseudoGoal> pseudo(new PseudoGoal(worker));
|
if (!worker.run(worker.makeRealisationGoal(nePath)))
|
||||||
worker.addRealisationGoal(nePath, pseudo);
|
|
||||||
worker.run();
|
|
||||||
|
|
||||||
if (!pseudo->isOkay())
|
|
||||||
throw Error(format("realisation of closure `%1%' failed") % nePath);
|
throw Error(format("realisation of closure `%1%' failed") % nePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1866,10 +1891,6 @@ void ensurePath(const Path & path)
|
||||||
if (isValidPath(path)) return;
|
if (isValidPath(path)) return;
|
||||||
|
|
||||||
Worker worker;
|
Worker worker;
|
||||||
shared_ptr<PseudoGoal> pseudo(new PseudoGoal(worker));
|
if (!worker.run(worker.makeSubstitutionGoal(path)))
|
||||||
worker.addSubstitutionGoal(path, pseudo);
|
|
||||||
worker.run();
|
|
||||||
|
|
||||||
if (!pseudo->isOkay())
|
|
||||||
throw Error(format("path `%1%' does not exist and cannot be created") % path);
|
throw Error(format("path `%1%' does not exist and cannot be created") % path);
|
||||||
}
|
}
|
||||||
|
|
|
@ -492,20 +492,19 @@ void Pid::kill()
|
||||||
{
|
{
|
||||||
if (pid == -1) return;
|
if (pid == -1) return;
|
||||||
|
|
||||||
printMsg(lvlError, format("killing child process %1%") % pid);
|
printMsg(lvlError, format("killing process %1%") % pid);
|
||||||
|
|
||||||
/* Send a KILL signal to the child. If it has its own process
|
/* Send a KILL signal to the child. If it has its own process
|
||||||
group, send the signal to every process in the child process
|
group, send the signal to every process in the child process
|
||||||
group (which hopefully includes *all* its children). */
|
group (which hopefully includes *all* its children). */
|
||||||
if (::kill(separatePG ? -pid : pid, SIGKILL) != 0)
|
if (::kill(separatePG ? -pid : pid, SIGKILL) != 0)
|
||||||
printMsg(lvlError, format("killing process %1%") % pid);
|
printMsg(lvlError, (SysError(format("killing process %1%") % pid).msg()));
|
||||||
else {
|
|
||||||
/* Wait until the child dies, disregarding the exit status. */
|
/* Wait until the child dies, disregarding the exit status. */
|
||||||
int status;
|
int status;
|
||||||
while (waitpid(pid, &status, 0) == -1)
|
while (waitpid(pid, &status, 0) == -1)
|
||||||
if (errno != EINTR) printMsg(lvlError,
|
if (errno != EINTR) printMsg(lvlError,
|
||||||
format("waiting for process %1%") % pid);
|
(SysError(format("waiting for process %1%") % pid).msg()));
|
||||||
}
|
|
||||||
|
|
||||||
pid = -1;
|
pid = -1;
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,9 +21,8 @@ substitutes.sh: substitutes.nix substituter.nix
|
||||||
substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix
|
substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix
|
||||||
fall-back.sh: fall-back.nix
|
fall-back.sh: fall-back.nix
|
||||||
|
|
||||||
#TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \
|
TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \
|
||||||
# build-hook.sh substitutes.sh substitutes2.sh
|
build-hook.sh substitutes.sh substitutes2.sh
|
||||||
TESTS = init.sh fall-back.sh
|
|
||||||
|
|
||||||
XFAIL_TESTS =
|
XFAIL_TESTS =
|
||||||
|
|
||||||
|
|
|
@ -2,11 +2,12 @@ storeExpr=$($TOP/src/nix-instantiate/nix-instantiate fall-back.nix)
|
||||||
|
|
||||||
echo "store expr is $storeExpr"
|
echo "store expr is $storeExpr"
|
||||||
|
|
||||||
# Register a non-existant successor.
|
# Register a non-existant successor (and a nox-existant substitute).
|
||||||
suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
|
suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
|
||||||
|
(echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute
|
||||||
$TOP/src/nix-store/nix-store --successor $storeExpr $suc
|
$TOP/src/nix-store/nix-store --successor $storeExpr $suc
|
||||||
|
|
||||||
outPath=$($TOP/src/nix-store/nix-store -qnfvvvvv "$storeExpr")
|
outPath=$($TOP/src/nix-store/nix-store -qnf "$storeExpr")
|
||||||
|
|
||||||
echo "output path is $outPath"
|
echo "output path is $outPath"
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue