From be81764320fc28131d23b85575076218eb7424c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 04:39:57 +0000 Subject: [PATCH 1/2] Factor out bits of the worker protocol to use elsewhere This introduces some shared infrastructure for our notion of protocols. We can then define multiple protocols in terms of that notion. We an also express how particular protocols depend on each other. For example, we can define a common protocol and a worker protocol, where the second depends on the first in terms of the data types it can read and write. The "serve" protocol can just use the common one for now, but will eventually need its own machinary just like the worker protocol for version-aware serialisers --- doc/internal-api/doxygen.cfg.in | 30 ++++ src/libstore/build/derivation-goal.cc | 10 +- src/libstore/common-protocol-impl.hh | 41 +++++ src/libstore/common-protocol.cc | 98 +++++++++++ src/libstore/common-protocol.hh | 106 ++++++++++++ src/libstore/derivations.cc | 12 +- src/libstore/export-import.cc | 12 +- src/libstore/legacy-ssh-store.cc | 44 ++--- .../length-prefixed-protocol-helper.hh | 162 ++++++++++++++++++ src/libstore/worker-protocol-impl.hh | 111 ++++-------- src/libstore/worker-protocol.cc | 81 +-------- src/libstore/worker-protocol.hh | 60 ++----- src/nix-store/nix-store.cc | 26 +-- 13 files changed, 542 insertions(+), 251 deletions(-) create mode 100644 src/libstore/common-protocol-impl.hh create mode 100644 src/libstore/common-protocol.cc create mode 100644 src/libstore/common-protocol.hh create mode 100644 src/libstore/length-prefixed-protocol-helper.hh diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 8f526536d..599be2470 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -54,6 +54,23 @@ INPUT = \ src/nix-env \ src/nix-store +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the # preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of @@ -61,3 +78,16 @@ INPUT = \ # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @RAPIDCHECK_HEADERS@ + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = \ + DECLARE_COMMON_SERIALISER \ + DECLARE_WORKER_SERIALISER \ + DECLARE_SERVE_SERIALISER \ + LENGTH_PREFIXED_PROTO_HELPER diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 83c0a3135..dc4d91079 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -8,8 +8,8 @@ #include "util.hh" #include "archive.hh" #include "compression.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -1185,11 +1185,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } - WorkerProto::WriteConn conn { hook->sink }; + CommonProto::WriteConn conn { hook->sink }; /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, conn, inputPaths); + CommonProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1200,7 +1200,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - WorkerProto::write(worker.store, conn, missingOutputs); + CommonProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh new file mode 100644 index 000000000..079c182b8 --- /dev/null +++ b/src/libstore/common-protocol-impl.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "common-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + + +/* protocol-specific templates */ + +} diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc new file mode 100644 index 000000000..f906814bc --- /dev/null +++ b/src/libstore/common-protocol.cc @@ -0,0 +1,98 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +/* protocol-agnostic definitions */ + +std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return readString(conn.from); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh new file mode 100644 index 000000000..f3f28972a --- /dev/null +++ b/src/libstore/common-protocol.hh @@ -0,0 +1,106 @@ +#pragma once +///@file + +#include "serialise.hh" + +namespace nix { + +class Store; +struct Source; + +// items being serialized +class StorePath; +struct ContentAddress; +struct DrvOutput; +struct Realisation; + + +/** + * Shared serializers between the worker protocol, serve protocol, and a + * few others. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct CommonProto +{ + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + }; + + template + struct Serialise; + + /** + * Wrapper function around `CommonProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, conn, t); + } +}; + +#define DECLARE_COMMON_SERIALISER(T) \ + struct CommonProto::Serialise< T > \ + { \ + static T read(const Store & store, CommonProto::ReadConn conn); \ + static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + } + +template<> +DECLARE_COMMON_SERIALISER(std::string); +template<> +DECLARE_COMMON_SERIALISER(StorePath); +template<> +DECLARE_COMMON_SERIALISER(ContentAddress); +template<> +DECLARE_COMMON_SERIALISER(DrvOutput); +template<> +DECLARE_COMMON_SERIALISER(Realisation); + +template +DECLARE_COMMON_SERIALISER(std::vector); +template +DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ + +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. + */ +template<> +DECLARE_COMMON_SERIALISER(std::optional); +template<> +DECLARE_COMMON_SERIALISER(std::optional); + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 67069c3c9..fc17e520c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,8 +4,8 @@ #include "globals.hh" #include "util.hh" #include "split.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "fs-accessor.hh" #include #include @@ -895,8 +895,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = in }); + drv.inputSrcs = CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -944,8 +944,8 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw); } - WorkerProto::write(store, - WorkerProto::WriteConn { .to = out }, + CommonProto::write(store, + CommonProto::WriteConn { .to = out }, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42..87b2f8741 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,8 +1,8 @@ #include "serialise.hh" #include "store-api.hh" #include "archive.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include @@ -46,8 +46,8 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, - WorkerProto::WriteConn { .to = teeSink }, + CommonProto::write(*this, + CommonProto::WriteConn { .to = teeSink }, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") @@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::Serialise::read(*this, - WorkerProto::ReadConn { .from = source }); + auto references = CommonProto::Serialise::read(*this, + CommonProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 78b05031a..7bf4476d0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,8 +6,8 @@ #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -50,37 +50,37 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor bool good = true; /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::ReadConn`. This makes it easy to use the + * factored out common protocol serialisers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::ReadConn () + operator CommonProto::ReadConn () { - return WorkerProto::ReadConn { + return CommonProto::ReadConn { .from = from, }; } /* - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::WriteConn`. This makes it easy to use the + * factored out common protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::WriteConn () + operator CommonProto::WriteConn () { - return WorkerProto::WriteConn { + return CommonProto::WriteConn { .to = to, }; } @@ -183,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->references = CommonProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -331,7 +331,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); + auto builtOutputs = CommonProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -409,10 +409,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : WorkerProto::Serialise::read(*this, *conn)) + for (auto & i : CommonProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +425,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + return CommonProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh new file mode 100644 index 000000000..4061b0cd6 --- /dev/null +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -0,0 +1,162 @@ +#pragma once +/** + * @file Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * Used by both the Worker and Serve protocols. + */ + +#include "types.hh" + +namespace nix { + +class Store; + +/** + * Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * @param T The type of the collection being serialised + * + * @param Inner This the most important parameter; this is the "inner" + * protocol. The user of this will substitute `MyProtocol` or similar + * when making a `MyProtocol::Serialiser>`. Note that the + * inside is allowed to call to call `Inner::Serialiser` on different + * types. This is especially important for `std::map` which doesn't have + * a single `T` but one `K` and one `V`. + */ +template +struct LengthPrefixedProtoHelper; + +/*! + * \typedef LengthPrefixedProtoHelper::S + * + * Read this as simply `using S = Inner::Serialise;`. + * + * It would be nice to use that directly, but C++ doesn't seem to allow + * it. The `typename` keyword needed to refer to `Inner` seems to greedy + * (low precedence), and then C++ complains that `Serialise` is not a + * type parameter but a real type. + * + * Making this `S` alias seems to be the only way to avoid these issues. + */ + +#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ + struct LengthPrefixedProtoHelper< Inner, T > \ + { \ + static T read(const Store & store, typename Inner::ReadConn conn); \ + static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + private: \ + template using S = typename Inner::template Serialise; \ + } + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); + +template +#define _X std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); +#undef _X + +template +std::vector +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::vector resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.push_back(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::set +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::set resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.insert(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::set & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::map +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::map resMap; + auto size = readNum(conn.from); + while (size--) { + auto k = S::read(store, conn); + auto v = S::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::map & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + S::write(store, conn, i.first); + S::write(store, conn, i.second); + } +} + +template +std::tuple +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + return std::tuple { + S::read(store, conn)..., + }; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (S::write(store, conn, args), ...); + }, res); +} + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index 4f797f95a..c043588d6 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -9,86 +9,51 @@ */ #include "worker-protocol.hh" +#include "length-prefixed-protocol-helper.hh" namespace nix { +/* protocol-agnostic templates */ + +#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ template -std::vector WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +struct WorkerProto::Serialise { - std::vector resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.push_back(WorkerProto::Serialise::read(store, conn)); + static T read(const Store & store, WorkerProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); } -} +}; -template -std::set WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::set resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.insert(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::set & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); - } -} - -template -std::map WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::map resMap; - auto size = readNum(conn.from); - while (size--) { - auto k = WorkerProto::Serialise::read(store, conn); - auto v = WorkerProto::Serialise::read(store, conn); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::map & resMap) -{ - conn.to << resMap.size(); - for (auto & i : resMap) { - WorkerProto::Serialise::write(store, conn, i.first); - WorkerProto::Serialise::write(store, conn, i.second); - } -} - -template -std::tuple WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return std::tuple { - WorkerProto::Serialise::read(store, conn)..., - }; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple & res) -{ - std::apply([&](const Us &... args) { - (WorkerProto::Serialise::write(store, conn, args), ...); - }, res); -} +/* protocol-specific templates */ } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a23130743..415e66f16 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -12,27 +12,7 @@ namespace nix { -std::string WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return readString(conn.from); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) -{ - conn.to << str; -} - - -StorePath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return store.parseStorePath(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) -{ - conn.to << store.printStorePath(storePath); -} - +/* protocol-specific definitions */ std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { @@ -68,17 +48,6 @@ void WorkerProto::Serialise>::write(const Store & sto } -ContentAddress WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) -{ - conn.to << renderContentAddress(ca); -} - - DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); @@ -91,32 +60,6 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -Realisation WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << realisation.toJSON().dump(); -} - - -DrvOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - - KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); @@ -168,26 +111,4 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - auto s = readString(conn.from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & storePathOpt) -{ - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parseOpt(readString(conn.from)); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & caOpt) -{ - conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); -} - } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b7f42f24d..c84060103 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "serialise.hh" +#include "common-protocol.hh" namespace nix { @@ -28,11 +28,7 @@ class Store; struct Source; // items being serialised -class StorePath; -struct ContentAddress; struct DerivedPath; -struct DrvOutput; -struct Realisation; struct BuildResult; struct KeyedBuildResult; enum TrustedFlag : bool; @@ -193,60 +189,32 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) * be legal specialization syntax. See below for what that looks like in * practice. */ -#define MAKE_WORKER_PROTO(T) \ - struct WorkerProto::Serialise< T > { \ +#define DECLARE_WORKER_SERIALISER(T) \ + struct WorkerProto::Serialise< T > \ + { \ static T read(const Store & store, WorkerProto::ReadConn conn); \ static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> -MAKE_WORKER_PROTO(std::string); +DECLARE_WORKER_SERIALISER(DerivedPath); template<> -MAKE_WORKER_PROTO(StorePath); +DECLARE_WORKER_SERIALISER(BuildResult); template<> -MAKE_WORKER_PROTO(ContentAddress); +DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> -MAKE_WORKER_PROTO(DerivedPath); -template<> -MAKE_WORKER_PROTO(DrvOutput); -template<> -MAKE_WORKER_PROTO(Realisation); -template<> -MAKE_WORKER_PROTO(BuildResult); -template<> -MAKE_WORKER_PROTO(KeyedBuildResult); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::optional); template -MAKE_WORKER_PROTO(std::vector); +DECLARE_WORKER_SERIALISER(std::vector); template -MAKE_WORKER_PROTO(std::set); +DECLARE_WORKER_SERIALISER(std::set); template -MAKE_WORKER_PROTO(std::tuple); +DECLARE_WORKER_SERIALISER(std::tuple); +#define COMMA_ , template -#define X_ std::map -MAKE_WORKER_PROTO(X_); -#undef X_ - -/** - * These use the empty string for the null case, relying on the fact - * that the underlying types never serialise to the empty string. - * - * We do this instead of a generic std::optional instance because - * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - * the same reason, we don't have a std::variant instances (ordinal - * tags 0...n). - * - * We could the generic instances and then these as specializations for - * compatability, but that's proven a bit finnicky, and also makes the - * worker protocol harder to implement in other languages where such - * specializations may not be allowed. - */ -template<> -MAKE_WORKER_PROTO(std::optional); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 96c3f7d7e..6fc22214a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,8 +11,8 @@ #include "serve-protocol.hh" #include "shared.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -821,8 +821,8 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); - WorkerProto::ReadConn rconn { .from = in }; - WorkerProto::WriteConn wconn { .to = out }; + CommonProto::ReadConn rconn { .from = in }; + CommonProto::WriteConn wconn { .to = out }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,19 +876,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); + CommonProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + CommonProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::Serialise::read(*store, rconn), out); + store->exportPaths(CommonProto::Serialise::read(*store, rconn), out); break; } @@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); + CommonProto::write(*store, wconn, builtOutputs); } break; @@ -971,9 +971,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::Serialise::read(*store, rconn), + store->computeFSClosure(CommonProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - WorkerProto::write(*store, wconn, closure); + CommonProto::write(*store, wconn, closure); break; } @@ -988,7 +988,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.references = CommonProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); From 4de54b21907e3fc41b9e6250396eb01d2b10c4db Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 4 Oct 2023 23:27:50 -0400 Subject: [PATCH 2/2] Unit test the "common protocol" too Copy the relevant tests to ensure the new interfaces added in the last commit are tested. Perhaps I should try to deduplicat these tests some more. However its not clear how to do that outside of a big ugly C++ macro. https://github.com/google/googletest/blob/main/docs/advanced.md has some stuff but it is cumbersome and I didn't figure it out yet. This is done in a separate commit in order to be sure that the first commit really didn't change any behavior; if we changed the implementation and the tests at once, it would be harder to tell whether or not some behavioral changes slipped in what is supposed to be a "pure refactor". Co-Authored-By: Valentin Gagarin --- src/libstore/tests/characterization.hh | 23 +++ src/libstore/tests/common-protocol.cc | 152 ++++++++++++++++++ src/libstore/tests/protocol.hh | 88 ++++++++++ src/libstore/tests/worker-protocol.cc | 90 ++--------- .../common-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/common-protocol/drv-output.bin | Bin 0 -> 176 bytes .../optional-content-address.bin | Bin 0 -> 64 bytes .../common-protocol/optional-store-path.bin | Bin 0 -> 72 bytes .../libstore/common-protocol/realisation.bin | Bin 0 -> 520 bytes .../libstore/common-protocol/set.bin | Bin 0 -> 152 bytes .../libstore/common-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/common-protocol/string.bin | Bin 0 -> 88 bytes .../libstore/common-protocol/vector.bin | Bin 0 -> 152 bytes 13 files changed, 280 insertions(+), 73 deletions(-) create mode 100644 src/libstore/tests/characterization.hh create mode 100644 src/libstore/tests/common-protocol.cc create mode 100644 src/libstore/tests/protocol.hh create mode 100644 unit-test-data/libstore/common-protocol/content-address.bin create mode 100644 unit-test-data/libstore/common-protocol/drv-output.bin create mode 100644 unit-test-data/libstore/common-protocol/optional-content-address.bin create mode 100644 unit-test-data/libstore/common-protocol/optional-store-path.bin create mode 100644 unit-test-data/libstore/common-protocol/realisation.bin create mode 100644 unit-test-data/libstore/common-protocol/set.bin create mode 100644 unit-test-data/libstore/common-protocol/store-path.bin create mode 100644 unit-test-data/libstore/common-protocol/string.bin create mode 100644 unit-test-data/libstore/common-protocol/vector.bin diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh new file mode 100644 index 000000000..5f366cb42 --- /dev/null +++ b/src/libstore/tests/characterization.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +namespace nix { + +/** + * The path to the `unit-test-data` directory. See the contributing + * guide in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +} diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc new file mode 100644 index 000000000..de57211f0 --- /dev/null +++ b/src/libstore/tests/common-protocol.cc @@ -0,0 +1,152 @@ +#include + +#include +#include + +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "common-protocol"; + +using CommonProtoTest = ProtoTest; + +CHARACTERIZATION_TEST( + CommonProtoTest, + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh new file mode 100644 index 000000000..86a900757 --- /dev/null +++ b/src/libstore/tests/protocol.hh @@ -0,0 +1,88 @@ +#include +#include + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +template +class ProtoTest : public LibStoreTest +{ + /** + * Read this as simply `using S = Inner::Serialise;`. + * + * See `LengthPrefixedProtoHelper::S` for the same trick, and its + * rationale. + */ + template using S = typename Proto::template Serialise; + +public: + Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem + ".bin"; + } + + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + S::read( + *store, + typename Proto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + Proto::write( + *store, + typename Proto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VALUE) \ + TEST_F(FIXTURE, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(FIXTURE, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } + +} diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index fa7cbe121..59d7ff96d 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -7,85 +7,17 @@ #include "worker-protocol-impl.hh" #include "derived-path.hh" #include "build-result.hh" -#include "tests/libstore.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" namespace nix { -class WorkerProtoTest : public LibStoreTest -{ -public: - Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol"; +const char workerProtoDir[] = "worker-protocol"; - bool testAccept() { - return getEnv("_NIX_TEST_ACCEPT") == "1"; - } - - Path goldenMaster(std::string_view testStem) { - return unitTestData + "/" + testStem + ".bin"; - } - - /** - * Golden test for `T` reading - */ - template - void readTest(PathView testStem, T value) - { - if (testAccept()) - { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; - } - else - { - auto expected = readFile(goldenMaster(testStem)); - - T got = ({ - StringSource from { expected }; - WorkerProto::Serialise::read( - *store, - WorkerProto::ReadConn { .from = from }); - }); - - ASSERT_EQ(got, value); - } - } - - /** - * Golden test for `T` write - */ - template - void writeTest(PathView testStem, const T & value) - { - auto file = goldenMaster(testStem); - - StringSink to; - WorkerProto::write( - *store, - WorkerProto::WriteConn { .to = to }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } - } -}; - -#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ - TEST_F(WorkerProtoTest, NAME ## _read) { \ - readTest(STEM, VALUE); \ - } \ - TEST_F(WorkerProtoTest, NAME ## _write) { \ - writeTest(STEM, VALUE); \ - } +using WorkerProtoTest = ProtoTest; CHARACTERIZATION_TEST( + WorkerProtoTest, string, "string", (std::tuple { @@ -97,6 +29,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, storePath, "store-path", (std::tuple { @@ -105,6 +38,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, contentAddress, "content-address", (std::tuple { @@ -123,6 +57,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, derivedPath, "derived-path", (std::tuple { @@ -138,6 +73,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, drvOutput, "drv-output", (std::tuple { @@ -152,6 +88,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, realisation, "realisation", (std::tuple { @@ -183,6 +120,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, buildResult, "build-result", ({ @@ -240,6 +178,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, keyedBuildResult, "keyed-build-result", ({ @@ -275,6 +214,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalTrustedFlag, "optional-trusted-flag", (std::tuple, std::optional, std::optional> { @@ -284,6 +224,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, vector, "vector", (std::tuple, std::vector, std::vector, std::vector>> { @@ -294,6 +235,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, set, "set", (std::tuple, std::set, std::set, std::set>> { @@ -304,6 +246,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalStorePath, "optional-store-path", (std::tuple, std::optional> { @@ -314,6 +257,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalContentAddress, "optional-content-address", (std::tuple, std::optional> { diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/unit-test-data/libstore/common-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/unit-test-data/libstore/common-protocol/drv-output.bin new file mode 100644 index 0000000000000000000000000000000000000000..800a45fd8757a15e8810066aed48be56a600b7d3 GIT binary patch literal 176 zcmY++xeWp_5I|8{*$(Wn=b{B@!gGlfp_LHTb8N(qe)KM%U7Sq@}q)U|6h9n2j1!MT?w+C~^gO WO9=?G&ojrYjy`x4ZufnEe#tjMpDPdm literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/unit-test-data/libstore/common-protocol/optional-content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe65ba27fd78ec7787eed7ebec8dfdead2fba GIT binary patch literal 64 zcmZQzfB#ECTBU06lLKJOBUy literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/unit-test-data/libstore/common-protocol/realisation.bin new file mode 100644 index 0000000000000000000000000000000000000000..2176c6c4afd96b32fe372de6084ac7f4c7a11d49 GIT binary patch literal 520 zcmdUrO-{o=429t+opq7s&z_l_03#3x z?rW}!>Uc!$VBY>Sr2mUC`<2<)qY;)1KN Po44)luetw6d9AunL$;dR literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/set.bin b/unit-test-data/libstore/common-protocol/set.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce11ede7fe7aa061a30b211a18cd7a336f879de8 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obCnXkvMPU52{CpHXOdBEdVDg4g4KThDln(&+h63;a literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/unit-test-data/libstore/common-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/string.bin b/unit-test-data/libstore/common-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H(