2020-10-11 19:17:24 +03:00
|
|
|
#pragma once
|
2023-04-01 06:18:41 +03:00
|
|
|
///@file
|
2020-10-11 19:17:24 +03:00
|
|
|
|
2020-10-12 20:16:00 +03:00
|
|
|
#include "types.hh"
|
|
|
|
#include "store-api.hh"
|
2022-03-08 20:50:46 +02:00
|
|
|
#include "build-result.hh"
|
2009-01-12 18:30:32 +02:00
|
|
|
|
2006-09-05 00:06:23 +03:00
|
|
|
namespace nix {
|
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Forward definition.
|
|
|
|
*/
|
2020-10-12 20:16:00 +03:00
|
|
|
struct Goal;
|
2020-12-01 15:57:56 +02:00
|
|
|
class Worker;
|
2004-06-18 21:09:32 +03:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* A pointer to a goal.
|
|
|
|
*/
|
2014-03-30 01:49:23 +02:00
|
|
|
typedef std::shared_ptr<Goal> GoalPtr;
|
|
|
|
typedef std::weak_ptr<Goal> WeakGoalPtr;
|
2004-06-18 21:09:32 +03:00
|
|
|
|
2014-11-24 17:48:04 +02:00
|
|
|
struct CompareGoalPtrs {
|
2017-12-11 20:05:14 +02:00
|
|
|
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
|
2014-11-24 17:48:04 +02:00
|
|
|
};
|
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Set of goals.
|
|
|
|
*/
|
2022-02-21 17:28:23 +02:00
|
|
|
typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
|
|
|
|
typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
|
2004-06-18 21:09:32 +03:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* A map of paths to goals (and the other way around).
|
|
|
|
*/
|
2019-12-05 20:11:09 +02:00
|
|
|
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
|
2004-06-18 21:09:32 +03:00
|
|
|
|
2023-05-08 21:45:46 +03:00
|
|
|
/**
|
|
|
|
* Used as a hint to the worker on how to schedule a particular goal. For example,
|
|
|
|
* builds are typically CPU- and memory-bound, while substitutions are I/O bound.
|
|
|
|
* Using this information, the worker might decide to schedule more or fewer goals
|
|
|
|
* of each category in parallel.
|
|
|
|
*/
|
|
|
|
enum struct JobCategory {
|
|
|
|
Build,
|
|
|
|
Substitution,
|
|
|
|
};
|
|
|
|
|
2020-06-15 20:25:35 +03:00
|
|
|
struct Goal : public std::enable_shared_from_this<Goal>
|
2004-06-18 21:09:32 +03:00
|
|
|
{
|
2013-01-02 13:38:28 +02:00
|
|
|
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
|
2012-07-27 16:59:18 +03:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Backlink to the worker.
|
|
|
|
*/
|
2004-06-18 21:09:32 +03:00
|
|
|
Worker & worker;
|
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Goals that this goal is waiting for.
|
|
|
|
*/
|
2004-06-25 18:36:09 +03:00
|
|
|
Goals waitees;
|
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Goals waiting for this one to finish. Must use weak pointers
|
|
|
|
* here to prevent cycles.
|
|
|
|
*/
|
2004-06-25 18:36:09 +03:00
|
|
|
WeakGoals waiters;
|
2004-06-18 21:09:32 +03:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Number of goals we are/were waiting for that have failed.
|
|
|
|
*/
|
2022-03-25 00:09:43 +02:00
|
|
|
size_t nrFailed = 0;
|
2004-06-25 13:21:44 +03:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Number of substitution goals we are/were waiting for that
|
|
|
|
* failed because there are no substituters.
|
|
|
|
*/
|
2022-03-25 00:09:43 +02:00
|
|
|
size_t nrNoSubstituters = 0;
|
2012-07-09 01:39:24 +03:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Number of substitution goals we are/were waiting for that
|
|
|
|
* failed because they had unsubstitutable references.
|
|
|
|
*/
|
2022-03-25 00:09:43 +02:00
|
|
|
size_t nrIncompleteClosure = 0;
|
2013-01-02 13:38:28 +02:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Name of this goal for debugging purposes.
|
|
|
|
*/
|
2022-02-25 17:00:00 +02:00
|
|
|
std::string name;
|
2005-02-18 11:50:20 +02:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Whether the goal is finished.
|
|
|
|
*/
|
2022-03-25 00:09:43 +02:00
|
|
|
ExitCode exitCode = ecBusy;
|
2005-02-23 13:19:27 +02:00
|
|
|
|
Make `KeyedBuildResult`, `BuildResult` like before, and fix bug another way
In https://github.com/NixOS/nix/pull/6311#discussion_r834863823, I
realized since derivation goals' wanted outputs can "grow" due to
overlapping dependencies (See `DerivationGoal::addWantedOutputs`, called
by `Worker::makeDerivationGoalCommon`), the previous bug fix had an
unfortunate side effect of causing more pointless rebuilds.
In paticular, we have this situation:
1. Goal made from `DerivedPath::Built { foo, {a} }`.
2. Goal gives on on substituting, starts building.
3. Goal made from `DerivedPath::Built { foo, {b} }`, in fact is just
modified original goal.
4. Though the goal had gotten as far as building, so all outputs were
going to be produced, `addWantedOutputs` no longer knows that and so
the goal is flagged to be restarted.
This might sound far-fetched with input-addressed drvs, where we usually
basically have all our goals "planned out" before we start doing
anything, but with CA derivation goals and especially RFC 92, where *drv
resolution* means goals are created after some building is completed, it
is more likely to happen.
So the first thing to do was restore the clearing of `wantedOutputs` we
used to do, and then filter the outputs in `buildPathsWithResults` to
only get the ones we care about.
But fix also has its own side effect in that the `DerivedPath` in the
`BuildResult` in `DerivationGoal` cannot be trusted; it is merely the
*first* `DerivedPath` for which this goal was originally created.
To remedy this, I made `BuildResult` be like it was before, and instead
made `KeyedBuildResult` be a subclass wit the path. Only
`buildPathsWithResults` returns `KeyedBuildResult`s, everything else
just becomes like it was before, where the "key" is unambiguous from
context.
I think separating the "primary key" field(s) from the other fields is
good practical in general anyways. (I would like to do the same thing
for `ValidPathInfo`.) Among other things, it allows constructions like
`std::map<Key, ThingWithKey>` where doesn't contain duplicate keys and
just precludes the possibility of those duplicate keys being out of
sync.
We might leverage the above someday to overload `buildPathsWithResults`
to take a *set* of return a *map* per the above.
-----
Unfortunately, we need to avoid C++20 strictness on designated
initializers.
(BTW
https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html
this offers some new syntax for this use-case. Hopefully this will be
adopted and we can eventually use it.)
No having that yet, maybe it would be better to not make
`KeyedBuildResult` a subclass to just avoid this.
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
2022-03-25 03:26:07 +02:00
|
|
|
protected:
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Build result.
|
|
|
|
*/
|
2022-03-08 20:50:46 +02:00
|
|
|
BuildResult buildResult;
|
|
|
|
|
Make `KeyedBuildResult`, `BuildResult` like before, and fix bug another way
In https://github.com/NixOS/nix/pull/6311#discussion_r834863823, I
realized since derivation goals' wanted outputs can "grow" due to
overlapping dependencies (See `DerivationGoal::addWantedOutputs`, called
by `Worker::makeDerivationGoalCommon`), the previous bug fix had an
unfortunate side effect of causing more pointless rebuilds.
In paticular, we have this situation:
1. Goal made from `DerivedPath::Built { foo, {a} }`.
2. Goal gives on on substituting, starts building.
3. Goal made from `DerivedPath::Built { foo, {b} }`, in fact is just
modified original goal.
4. Though the goal had gotten as far as building, so all outputs were
going to be produced, `addWantedOutputs` no longer knows that and so
the goal is flagged to be restarted.
This might sound far-fetched with input-addressed drvs, where we usually
basically have all our goals "planned out" before we start doing
anything, but with CA derivation goals and especially RFC 92, where *drv
resolution* means goals are created after some building is completed, it
is more likely to happen.
So the first thing to do was restore the clearing of `wantedOutputs` we
used to do, and then filter the outputs in `buildPathsWithResults` to
only get the ones we care about.
But fix also has its own side effect in that the `DerivedPath` in the
`BuildResult` in `DerivationGoal` cannot be trusted; it is merely the
*first* `DerivedPath` for which this goal was originally created.
To remedy this, I made `BuildResult` be like it was before, and instead
made `KeyedBuildResult` be a subclass wit the path. Only
`buildPathsWithResults` returns `KeyedBuildResult`s, everything else
just becomes like it was before, where the "key" is unambiguous from
context.
I think separating the "primary key" field(s) from the other fields is
good practical in general anyways. (I would like to do the same thing
for `ValidPathInfo`.) Among other things, it allows constructions like
`std::map<Key, ThingWithKey>` where doesn't contain duplicate keys and
just precludes the possibility of those duplicate keys being out of
sync.
We might leverage the above someday to overload `buildPathsWithResults`
to take a *set* of return a *map* per the above.
-----
Unfortunately, we need to avoid C++20 strictness on designated
initializers.
(BTW
https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html
this offers some new syntax for this use-case. Hopefully this will be
adopted and we can eventually use it.)
No having that yet, maybe it would be better to not make
`KeyedBuildResult` a subclass to just avoid this.
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
2022-03-25 03:26:07 +02:00
|
|
|
public:
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Project a `BuildResult` with just the information that pertains
|
|
|
|
* to the given request.
|
|
|
|
*
|
|
|
|
* In general, goals may be aliased between multiple requests, and
|
|
|
|
* the stored `BuildResult` has information for the union of all
|
|
|
|
* requests. We don't want to leak what the other request are for
|
|
|
|
* sake of both privacy and determinism, and this "safe accessor"
|
|
|
|
* ensures we don't.
|
|
|
|
*/
|
|
|
|
BuildResult getBuildResult(const DerivedPath &);
|
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Exception containing an error message, if any.
|
|
|
|
*/
|
2020-06-15 20:25:35 +03:00
|
|
|
std::optional<Error> ex;
|
|
|
|
|
2022-03-09 13:25:35 +02:00
|
|
|
Goal(Worker & worker, DerivedPath path)
|
|
|
|
: worker(worker)
|
2022-03-25 00:09:43 +02:00
|
|
|
{ }
|
2004-06-18 21:09:32 +03:00
|
|
|
|
|
|
|
virtual ~Goal()
|
|
|
|
{
|
2005-02-18 11:50:20 +02:00
|
|
|
trace("goal destroyed");
|
2004-06-18 21:09:32 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual void work() = 0;
|
|
|
|
|
2004-06-25 18:36:09 +03:00
|
|
|
void addWaitee(GoalPtr waitee);
|
2004-06-18 21:09:32 +03:00
|
|
|
|
2006-12-08 19:26:21 +02:00
|
|
|
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
|
2004-06-25 13:21:44 +03:00
|
|
|
|
2022-02-25 17:00:00 +02:00
|
|
|
virtual void handleChildOutput(int fd, std::string_view data)
|
2005-10-17 18:33:24 +03:00
|
|
|
{
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual void handleEOF(int fd)
|
2004-06-29 12:41:50 +03:00
|
|
|
{
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
|
2023-03-02 16:44:19 +02:00
|
|
|
void trace(std::string_view s);
|
2005-02-18 11:50:20 +02:00
|
|
|
|
2022-02-25 17:00:00 +02:00
|
|
|
std::string getName()
|
2005-02-18 11:50:20 +02:00
|
|
|
{
|
|
|
|
return name;
|
|
|
|
}
|
2012-07-27 16:59:18 +03:00
|
|
|
|
2023-04-07 16:55:28 +03:00
|
|
|
/**
|
|
|
|
* Callback in case of a timeout. It should wake up its waiters,
|
|
|
|
* get rid of any running child processes that are being monitored
|
|
|
|
* by the worker (important!), etc.
|
|
|
|
*/
|
2020-06-15 20:25:35 +03:00
|
|
|
virtual void timedOut(Error && ex) = 0;
|
2006-12-08 19:26:21 +02:00
|
|
|
|
2022-02-25 17:00:00 +02:00
|
|
|
virtual std::string key() = 0;
|
2014-11-24 17:48:04 +02:00
|
|
|
|
2020-06-15 20:25:35 +03:00
|
|
|
void amDone(ExitCode result, std::optional<Error> ex = {});
|
2021-04-07 13:21:31 +03:00
|
|
|
|
|
|
|
virtual void cleanup() { }
|
2023-05-08 21:45:46 +03:00
|
|
|
|
|
|
|
virtual JobCategory jobCategory() = 0;
|
2004-06-18 21:09:32 +03:00
|
|
|
};
|
|
|
|
|
2020-10-11 19:17:24 +03:00
|
|
|
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
|
2012-10-02 21:08:59 +03:00
|
|
|
|
2006-09-05 00:06:23 +03:00
|
|
|
}
|