diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 8f526536d..599be2470 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -54,6 +54,23 @@ INPUT = \ src/nix-env \ src/nix-store +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the # preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of @@ -61,3 +78,16 @@ INPUT = \ # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @RAPIDCHECK_HEADERS@ + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = \ + DECLARE_COMMON_SERIALISER \ + DECLARE_WORKER_SERIALISER \ + DECLARE_SERVE_SERIALISER \ + LENGTH_PREFIXED_PROTO_HELPER diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 83c0a3135..dc4d91079 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -8,8 +8,8 @@ #include "util.hh" #include "archive.hh" #include "compression.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -1185,11 +1185,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } - WorkerProto::WriteConn conn { hook->sink }; + CommonProto::WriteConn conn { hook->sink }; /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, conn, inputPaths); + CommonProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1200,7 +1200,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - WorkerProto::write(worker.store, conn, missingOutputs); + CommonProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh new file mode 100644 index 000000000..079c182b8 --- /dev/null +++ b/src/libstore/common-protocol-impl.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "common-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + + +/* protocol-specific templates */ + +} diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc new file mode 100644 index 000000000..f906814bc --- /dev/null +++ b/src/libstore/common-protocol.cc @@ -0,0 +1,98 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +/* protocol-agnostic definitions */ + +std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return readString(conn.from); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh new file mode 100644 index 000000000..f3f28972a --- /dev/null +++ b/src/libstore/common-protocol.hh @@ -0,0 +1,106 @@ +#pragma once +///@file + +#include "serialise.hh" + +namespace nix { + +class Store; +struct Source; + +// items being serialized +class StorePath; +struct ContentAddress; +struct DrvOutput; +struct Realisation; + + +/** + * Shared serializers between the worker protocol, serve protocol, and a + * few others. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct CommonProto +{ + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + }; + + template + struct Serialise; + + /** + * Wrapper function around `CommonProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, conn, t); + } +}; + +#define DECLARE_COMMON_SERIALISER(T) \ + struct CommonProto::Serialise< T > \ + { \ + static T read(const Store & store, CommonProto::ReadConn conn); \ + static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + } + +template<> +DECLARE_COMMON_SERIALISER(std::string); +template<> +DECLARE_COMMON_SERIALISER(StorePath); +template<> +DECLARE_COMMON_SERIALISER(ContentAddress); +template<> +DECLARE_COMMON_SERIALISER(DrvOutput); +template<> +DECLARE_COMMON_SERIALISER(Realisation); + +template +DECLARE_COMMON_SERIALISER(std::vector); +template +DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ + +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. + */ +template<> +DECLARE_COMMON_SERIALISER(std::optional); +template<> +DECLARE_COMMON_SERIALISER(std::optional); + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 67069c3c9..fc17e520c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,8 +4,8 @@ #include "globals.hh" #include "util.hh" #include "split.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "fs-accessor.hh" #include #include @@ -895,8 +895,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = in }); + drv.inputSrcs = CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -944,8 +944,8 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw); } - WorkerProto::write(store, - WorkerProto::WriteConn { .to = out }, + CommonProto::write(store, + CommonProto::WriteConn { .to = out }, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42..87b2f8741 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,8 +1,8 @@ #include "serialise.hh" #include "store-api.hh" #include "archive.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include @@ -46,8 +46,8 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, - WorkerProto::WriteConn { .to = teeSink }, + CommonProto::write(*this, + CommonProto::WriteConn { .to = teeSink }, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") @@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::Serialise::read(*this, - WorkerProto::ReadConn { .from = source }); + auto references = CommonProto::Serialise::read(*this, + CommonProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 78b05031a..7bf4476d0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,8 +6,8 @@ #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -50,37 +50,37 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor bool good = true; /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::ReadConn`. This makes it easy to use the + * factored out common protocol serialisers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::ReadConn () + operator CommonProto::ReadConn () { - return WorkerProto::ReadConn { + return CommonProto::ReadConn { .from = from, }; } /* - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::WriteConn`. This makes it easy to use the + * factored out common protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::WriteConn () + operator CommonProto::WriteConn () { - return WorkerProto::WriteConn { + return CommonProto::WriteConn { .to = to, }; } @@ -183,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->references = CommonProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -331,7 +331,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); + auto builtOutputs = CommonProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -409,10 +409,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : WorkerProto::Serialise::read(*this, *conn)) + for (auto & i : CommonProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +425,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + return CommonProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh new file mode 100644 index 000000000..4061b0cd6 --- /dev/null +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -0,0 +1,162 @@ +#pragma once +/** + * @file Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * Used by both the Worker and Serve protocols. + */ + +#include "types.hh" + +namespace nix { + +class Store; + +/** + * Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * @param T The type of the collection being serialised + * + * @param Inner This the most important parameter; this is the "inner" + * protocol. The user of this will substitute `MyProtocol` or similar + * when making a `MyProtocol::Serialiser>`. Note that the + * inside is allowed to call to call `Inner::Serialiser` on different + * types. This is especially important for `std::map` which doesn't have + * a single `T` but one `K` and one `V`. + */ +template +struct LengthPrefixedProtoHelper; + +/*! + * \typedef LengthPrefixedProtoHelper::S + * + * Read this as simply `using S = Inner::Serialise;`. + * + * It would be nice to use that directly, but C++ doesn't seem to allow + * it. The `typename` keyword needed to refer to `Inner` seems to greedy + * (low precedence), and then C++ complains that `Serialise` is not a + * type parameter but a real type. + * + * Making this `S` alias seems to be the only way to avoid these issues. + */ + +#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ + struct LengthPrefixedProtoHelper< Inner, T > \ + { \ + static T read(const Store & store, typename Inner::ReadConn conn); \ + static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + private: \ + template using S = typename Inner::template Serialise; \ + } + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); + +template +#define _X std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); +#undef _X + +template +std::vector +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::vector resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.push_back(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::set +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::set resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.insert(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::set & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::map +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::map resMap; + auto size = readNum(conn.from); + while (size--) { + auto k = S::read(store, conn); + auto v = S::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::map & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + S::write(store, conn, i.first); + S::write(store, conn, i.second); + } +} + +template +std::tuple +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + return std::tuple { + S::read(store, conn)..., + }; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (S::write(store, conn, args), ...); + }, res); +} + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index 4f797f95a..c043588d6 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -9,86 +9,51 @@ */ #include "worker-protocol.hh" +#include "length-prefixed-protocol-helper.hh" namespace nix { +/* protocol-agnostic templates */ + +#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ template -std::vector WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +struct WorkerProto::Serialise { - std::vector resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.push_back(WorkerProto::Serialise::read(store, conn)); + static T read(const Store & store, WorkerProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); } -} +}; -template -std::set WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::set resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.insert(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::set & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); - } -} - -template -std::map WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::map resMap; - auto size = readNum(conn.from); - while (size--) { - auto k = WorkerProto::Serialise::read(store, conn); - auto v = WorkerProto::Serialise::read(store, conn); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::map & resMap) -{ - conn.to << resMap.size(); - for (auto & i : resMap) { - WorkerProto::Serialise::write(store, conn, i.first); - WorkerProto::Serialise::write(store, conn, i.second); - } -} - -template -std::tuple WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return std::tuple { - WorkerProto::Serialise::read(store, conn)..., - }; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple & res) -{ - std::apply([&](const Us &... args) { - (WorkerProto::Serialise::write(store, conn, args), ...); - }, res); -} +/* protocol-specific templates */ } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a23130743..415e66f16 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -12,27 +12,7 @@ namespace nix { -std::string WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return readString(conn.from); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) -{ - conn.to << str; -} - - -StorePath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return store.parseStorePath(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) -{ - conn.to << store.printStorePath(storePath); -} - +/* protocol-specific definitions */ std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { @@ -68,17 +48,6 @@ void WorkerProto::Serialise>::write(const Store & sto } -ContentAddress WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) -{ - conn.to << renderContentAddress(ca); -} - - DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); @@ -91,32 +60,6 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -Realisation WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << realisation.toJSON().dump(); -} - - -DrvOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - - KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); @@ -168,26 +111,4 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - auto s = readString(conn.from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & storePathOpt) -{ - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parseOpt(readString(conn.from)); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & caOpt) -{ - conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); -} - } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b7f42f24d..c84060103 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "serialise.hh" +#include "common-protocol.hh" namespace nix { @@ -28,11 +28,7 @@ class Store; struct Source; // items being serialised -class StorePath; -struct ContentAddress; struct DerivedPath; -struct DrvOutput; -struct Realisation; struct BuildResult; struct KeyedBuildResult; enum TrustedFlag : bool; @@ -193,60 +189,32 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) * be legal specialization syntax. See below for what that looks like in * practice. */ -#define MAKE_WORKER_PROTO(T) \ - struct WorkerProto::Serialise< T > { \ +#define DECLARE_WORKER_SERIALISER(T) \ + struct WorkerProto::Serialise< T > \ + { \ static T read(const Store & store, WorkerProto::ReadConn conn); \ static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> -MAKE_WORKER_PROTO(std::string); +DECLARE_WORKER_SERIALISER(DerivedPath); template<> -MAKE_WORKER_PROTO(StorePath); +DECLARE_WORKER_SERIALISER(BuildResult); template<> -MAKE_WORKER_PROTO(ContentAddress); +DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> -MAKE_WORKER_PROTO(DerivedPath); -template<> -MAKE_WORKER_PROTO(DrvOutput); -template<> -MAKE_WORKER_PROTO(Realisation); -template<> -MAKE_WORKER_PROTO(BuildResult); -template<> -MAKE_WORKER_PROTO(KeyedBuildResult); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::optional); template -MAKE_WORKER_PROTO(std::vector); +DECLARE_WORKER_SERIALISER(std::vector); template -MAKE_WORKER_PROTO(std::set); +DECLARE_WORKER_SERIALISER(std::set); template -MAKE_WORKER_PROTO(std::tuple); +DECLARE_WORKER_SERIALISER(std::tuple); +#define COMMA_ , template -#define X_ std::map -MAKE_WORKER_PROTO(X_); -#undef X_ - -/** - * These use the empty string for the null case, relying on the fact - * that the underlying types never serialise to the empty string. - * - * We do this instead of a generic std::optional instance because - * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - * the same reason, we don't have a std::variant instances (ordinal - * tags 0...n). - * - * We could the generic instances and then these as specializations for - * compatability, but that's proven a bit finnicky, and also makes the - * worker protocol harder to implement in other languages where such - * specializations may not be allowed. - */ -template<> -MAKE_WORKER_PROTO(std::optional); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 96c3f7d7e..6fc22214a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,8 +11,8 @@ #include "serve-protocol.hh" #include "shared.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -821,8 +821,8 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); - WorkerProto::ReadConn rconn { .from = in }; - WorkerProto::WriteConn wconn { .to = out }; + CommonProto::ReadConn rconn { .from = in }; + CommonProto::WriteConn wconn { .to = out }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,19 +876,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); + CommonProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + CommonProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::Serialise::read(*store, rconn), out); + store->exportPaths(CommonProto::Serialise::read(*store, rconn), out); break; } @@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); + CommonProto::write(*store, wconn, builtOutputs); } break; @@ -971,9 +971,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::Serialise::read(*store, rconn), + store->computeFSClosure(CommonProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - WorkerProto::write(*store, wconn, closure); + CommonProto::write(*store, wconn, closure); break; } @@ -988,7 +988,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.references = CommonProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in));