2012-07-18 21:59:03 +03:00
|
|
|
#pragma once
|
2023-04-01 06:18:41 +03:00
|
|
|
///@file
|
2006-11-30 22:13:59 +02:00
|
|
|
|
2020-10-08 18:36:51 +03:00
|
|
|
#include "store-api.hh"
|
|
|
|
#include "serialise.hh"
|
|
|
|
|
2006-12-05 03:31:45 +02:00
|
|
|
namespace nix {
|
|
|
|
|
|
|
|
|
2007-09-18 12:11:20 +03:00
|
|
|
#define WORKER_MAGIC_1 0x6e697863
|
|
|
|
#define WORKER_MAGIC_2 0x6478696f
|
|
|
|
|
2022-12-26 22:21:08 +02:00
|
|
|
#define PROTOCOL_VERSION (1 << 8 | 35)
|
2007-09-18 12:11:20 +03:00
|
|
|
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
2007-11-16 18:15:26 +02:00
|
|
|
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
2006-11-30 22:13:59 +02:00
|
|
|
|
|
|
|
|
2023-03-27 04:12:25 +03:00
|
|
|
/**
|
|
|
|
* Enumeration of all the request types for the "worker protocol", used
|
|
|
|
* by unix:// and ssh-ng:// stores.
|
|
|
|
*/
|
2006-11-30 22:13:59 +02:00
|
|
|
typedef enum {
|
2008-06-18 12:34:17 +03:00
|
|
|
wopIsValidPath = 1,
|
2007-08-13 14:15:13 +03:00
|
|
|
wopHasSubstitutes = 3,
|
2016-04-19 19:50:15 +03:00
|
|
|
wopQueryPathHash = 4, // obsolete
|
|
|
|
wopQueryReferences = 5, // obsolete
|
2008-06-18 12:34:17 +03:00
|
|
|
wopQueryReferrers = 6,
|
|
|
|
wopAddToStore = 7,
|
2020-09-17 21:19:15 +03:00
|
|
|
wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore
|
2012-06-27 23:58:15 +03:00
|
|
|
wopBuildPaths = 9,
|
2008-06-18 12:34:17 +03:00
|
|
|
wopEnsurePath = 10,
|
|
|
|
wopAddTempRoot = 11,
|
|
|
|
wopAddIndirectRoot = 12,
|
|
|
|
wopSyncWithGC = 13,
|
|
|
|
wopFindRoots = 14,
|
2016-05-04 14:36:54 +03:00
|
|
|
wopExportPath = 16, // obsolete
|
2016-04-19 19:50:15 +03:00
|
|
|
wopQueryDeriver = 18, // obsolete
|
2008-06-18 12:34:17 +03:00
|
|
|
wopSetOptions = 19,
|
|
|
|
wopCollectGarbage = 20,
|
2008-08-04 14:44:50 +03:00
|
|
|
wopQuerySubstitutablePathInfo = 21,
|
2020-06-10 12:20:52 +03:00
|
|
|
wopQueryDerivationOutputs = 22, // obsolete
|
2012-07-11 17:49:04 +03:00
|
|
|
wopQueryAllValidPaths = 23,
|
2010-05-04 13:45:10 +03:00
|
|
|
wopQueryFailedPaths = 24,
|
|
|
|
wopClearFailedPaths = 25,
|
2010-11-17 14:08:01 +02:00
|
|
|
wopQueryPathInfo = 26,
|
2016-05-04 14:36:54 +03:00
|
|
|
wopImportPaths = 27, // obsolete
|
2020-06-12 13:46:33 +03:00
|
|
|
wopQueryDerivationOutputNames = 28, // obsolete
|
2012-07-18 01:55:39 +03:00
|
|
|
wopQueryPathFromHashPart = 29,
|
2012-07-18 17:47:59 +03:00
|
|
|
wopQuerySubstitutablePathInfos = 30,
|
|
|
|
wopQueryValidPaths = 31,
|
|
|
|
wopQuerySubstitutablePaths = 32,
|
2012-12-20 19:41:44 +02:00
|
|
|
wopQueryValidDerivers = 33,
|
2015-06-02 00:20:11 +03:00
|
|
|
wopOptimiseStore = 34,
|
2015-09-03 13:56:59 +03:00
|
|
|
wopVerifyStore = 35,
|
|
|
|
wopBuildDerivation = 36,
|
2016-04-05 16:30:22 +03:00
|
|
|
wopAddSignatures = 37,
|
2016-09-02 21:33:58 +03:00
|
|
|
wopNarFromPath = 38,
|
2017-04-06 19:40:19 +03:00
|
|
|
wopAddToStoreNar = 39,
|
|
|
|
wopQueryMissing = 40,
|
2020-06-10 12:20:52 +03:00
|
|
|
wopQueryDerivationOutputMap = 41,
|
2020-10-08 18:36:51 +03:00
|
|
|
wopRegisterDrvOutput = 42,
|
|
|
|
wopQueryRealisation = 43,
|
2021-07-26 14:31:09 +03:00
|
|
|
wopAddMultipleToStore = 44,
|
2022-01-17 20:45:21 +02:00
|
|
|
wopAddBuildLog = 45,
|
2022-03-08 20:50:46 +02:00
|
|
|
wopBuildPathsWithResults = 46,
|
2006-11-30 22:13:59 +02:00
|
|
|
} WorkerOp;
|
|
|
|
|
|
|
|
|
2006-12-03 04:08:13 +02:00
|
|
|
#define STDERR_NEXT 0x6f6c6d67
|
2007-02-21 19:34:02 +02:00
|
|
|
#define STDERR_READ 0x64617461 // data needed from source
|
|
|
|
#define STDERR_WRITE 0x64617416 // data for sink
|
2006-12-03 04:08:13 +02:00
|
|
|
#define STDERR_LAST 0x616c7473
|
|
|
|
#define STDERR_ERROR 0x63787470
|
2017-08-28 19:49:42 +03:00
|
|
|
#define STDERR_START_ACTIVITY 0x53545254
|
|
|
|
#define STDERR_STOP_ACTIVITY 0x53544f50
|
|
|
|
#define STDERR_RESULT 0x52534c54
|
2006-12-03 04:08:13 +02:00
|
|
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
class Store;
|
2019-10-30 21:38:02 +02:00
|
|
|
struct Source;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
2023-03-27 04:12:25 +03:00
|
|
|
/**
|
|
|
|
* Used to guide overloading
|
|
|
|
*
|
|
|
|
* See https://en.cppreference.com/w/cpp/language/adl for the broader
|
|
|
|
* concept of what is going on here.
|
|
|
|
*/
|
2020-07-25 00:02:51 +03:00
|
|
|
template<typename T>
|
2020-09-30 03:39:06 +03:00
|
|
|
struct Phantom {};
|
|
|
|
|
|
|
|
|
|
|
|
namespace worker_proto {
|
|
|
|
/* FIXME maybe move more stuff inside here */
|
|
|
|
|
|
|
|
#define MAKE_WORKER_PROTO(TEMPLATE, T) \
|
|
|
|
TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
|
|
|
|
TEMPLATE void write(const Store & store, Sink & out, const T & str)
|
|
|
|
|
|
|
|
MAKE_WORKER_PROTO(, std::string);
|
|
|
|
MAKE_WORKER_PROTO(, StorePath);
|
|
|
|
MAKE_WORKER_PROTO(, ContentAddress);
|
2021-04-05 16:48:18 +03:00
|
|
|
MAKE_WORKER_PROTO(, DerivedPath);
|
2021-01-26 11:48:41 +02:00
|
|
|
MAKE_WORKER_PROTO(, Realisation);
|
|
|
|
MAKE_WORKER_PROTO(, DrvOutput);
|
2022-03-08 20:50:46 +02:00
|
|
|
MAKE_WORKER_PROTO(, BuildResult);
|
Make `KeyedBuildResult`, `BuildResult` like before, and fix bug another way
In https://github.com/NixOS/nix/pull/6311#discussion_r834863823, I
realized since derivation goals' wanted outputs can "grow" due to
overlapping dependencies (See `DerivationGoal::addWantedOutputs`, called
by `Worker::makeDerivationGoalCommon`), the previous bug fix had an
unfortunate side effect of causing more pointless rebuilds.
In paticular, we have this situation:
1. Goal made from `DerivedPath::Built { foo, {a} }`.
2. Goal gives on on substituting, starts building.
3. Goal made from `DerivedPath::Built { foo, {b} }`, in fact is just
modified original goal.
4. Though the goal had gotten as far as building, so all outputs were
going to be produced, `addWantedOutputs` no longer knows that and so
the goal is flagged to be restarted.
This might sound far-fetched with input-addressed drvs, where we usually
basically have all our goals "planned out" before we start doing
anything, but with CA derivation goals and especially RFC 92, where *drv
resolution* means goals are created after some building is completed, it
is more likely to happen.
So the first thing to do was restore the clearing of `wantedOutputs` we
used to do, and then filter the outputs in `buildPathsWithResults` to
only get the ones we care about.
But fix also has its own side effect in that the `DerivedPath` in the
`BuildResult` in `DerivationGoal` cannot be trusted; it is merely the
*first* `DerivedPath` for which this goal was originally created.
To remedy this, I made `BuildResult` be like it was before, and instead
made `KeyedBuildResult` be a subclass wit the path. Only
`buildPathsWithResults` returns `KeyedBuildResult`s, everything else
just becomes like it was before, where the "key" is unambiguous from
context.
I think separating the "primary key" field(s) from the other fields is
good practical in general anyways. (I would like to do the same thing
for `ValidPathInfo`.) Among other things, it allows constructions like
`std::map<Key, ThingWithKey>` where doesn't contain duplicate keys and
just precludes the possibility of those duplicate keys being out of
sync.
We might leverage the above someday to overload `buildPathsWithResults`
to take a *set* of return a *map* per the above.
-----
Unfortunately, we need to avoid C++20 strictness on designated
initializers.
(BTW
https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html
this offers some new syntax for this use-case. Hopefully this will be
adopted and we can eventually use it.)
No having that yet, maybe it would be better to not make
`KeyedBuildResult` a subclass to just avoid this.
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
2022-03-25 03:26:07 +02:00
|
|
|
MAKE_WORKER_PROTO(, KeyedBuildResult);
|
2022-12-26 22:21:08 +02:00
|
|
|
MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
|
2020-09-30 03:39:06 +03:00
|
|
|
|
2021-03-02 05:50:41 +02:00
|
|
|
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
|
2020-09-30 03:39:06 +03:00
|
|
|
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
|
2020-08-07 01:04:13 +03:00
|
|
|
|
2020-09-30 03:39:06 +03:00
|
|
|
#define X_ template<typename K, typename V>
|
|
|
|
#define Y_ std::map<K, V>
|
|
|
|
MAKE_WORKER_PROTO(X_, Y_);
|
|
|
|
#undef X_
|
|
|
|
#undef Y_
|
2020-07-25 00:02:51 +03:00
|
|
|
|
2023-03-27 04:12:25 +03:00
|
|
|
/**
|
|
|
|
* These use the empty string for the null case, relying on the fact
|
|
|
|
* that the underlying types never serialize to the empty string.
|
|
|
|
*
|
|
|
|
* We do this instead of a generic std::optional<T> instance because
|
|
|
|
* ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
|
|
|
|
* the same reason, we don't have a std::variant<T..> instances (ordinal
|
|
|
|
* tags 0...n).
|
|
|
|
*
|
|
|
|
* We could the generic instances and then these as specializations for
|
|
|
|
* compatability, but that's proven a bit finnicky, and also makes the
|
|
|
|
* worker protocol harder to implement in other languages where such
|
|
|
|
* specializations may not be allowed.
|
2020-10-07 15:48:35 +03:00
|
|
|
*/
|
|
|
|
MAKE_WORKER_PROTO(, std::optional<StorePath>);
|
|
|
|
MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
|
|
|
|
|
2021-03-02 05:50:41 +02:00
|
|
|
template<typename T>
|
|
|
|
std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
|
|
|
|
{
|
|
|
|
std::vector<T> resSet;
|
|
|
|
auto size = readNum<size_t>(from);
|
|
|
|
while (size--) {
|
|
|
|
resSet.push_back(read(store, from, Phantom<T> {}));
|
|
|
|
}
|
|
|
|
return resSet;
|
|
|
|
}
|
|
|
|
|
|
|
|
template<typename T>
|
|
|
|
void write(const Store & store, Sink & out, const std::vector<T> & resSet)
|
|
|
|
{
|
|
|
|
out << resSet.size();
|
|
|
|
for (auto & key : resSet) {
|
|
|
|
write(store, out, key);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-25 00:02:51 +03:00
|
|
|
template<typename T>
|
2020-09-30 03:39:06 +03:00
|
|
|
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
|
|
|
|
{
|
|
|
|
std::set<T> resSet;
|
|
|
|
auto size = readNum<size_t>(from);
|
|
|
|
while (size--) {
|
|
|
|
resSet.insert(read(store, from, Phantom<T> {}));
|
2020-07-25 00:02:51 +03:00
|
|
|
}
|
2020-09-30 03:39:06 +03:00
|
|
|
return resSet;
|
|
|
|
}
|
2020-07-25 00:02:51 +03:00
|
|
|
|
2020-09-30 03:39:06 +03:00
|
|
|
template<typename T>
|
|
|
|
void write(const Store & store, Sink & out, const std::set<T> & resSet)
|
|
|
|
{
|
|
|
|
out << resSet.size();
|
|
|
|
for (auto & key : resSet) {
|
|
|
|
write(store, out, key);
|
2020-08-04 22:02:05 +03:00
|
|
|
}
|
2020-09-30 03:39:06 +03:00
|
|
|
}
|
2020-08-04 22:02:05 +03:00
|
|
|
|
|
|
|
template<typename K, typename V>
|
2020-09-30 03:39:06 +03:00
|
|
|
std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
|
|
|
|
{
|
|
|
|
std::map<K, V> resMap;
|
|
|
|
auto size = readNum<size_t>(from);
|
|
|
|
while (size--) {
|
|
|
|
auto k = read(store, from, Phantom<K> {});
|
|
|
|
auto v = read(store, from, Phantom<V> {});
|
|
|
|
resMap.insert_or_assign(std::move(k), std::move(v));
|
2020-07-25 00:02:51 +03:00
|
|
|
}
|
2020-09-30 03:39:06 +03:00
|
|
|
return resMap;
|
|
|
|
}
|
2020-07-25 00:02:51 +03:00
|
|
|
|
2020-09-30 03:39:06 +03:00
|
|
|
template<typename K, typename V>
|
|
|
|
void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
|
|
|
|
{
|
|
|
|
out << resMap.size();
|
|
|
|
for (auto & i : resMap) {
|
|
|
|
write(store, out, i.first);
|
|
|
|
write(store, out, i.second);
|
2020-07-25 00:02:51 +03:00
|
|
|
}
|
2020-09-30 03:39:06 +03:00
|
|
|
}
|
2020-07-25 00:02:51 +03:00
|
|
|
|
2020-09-30 03:39:06 +03:00
|
|
|
}
|
2020-06-20 01:06:19 +03:00
|
|
|
|
2006-12-05 03:31:45 +02:00
|
|
|
}
|