diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 4e4372331..282b75af2 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -270,6 +270,9 @@ Derivations can declare some infrequently used optional attributes. useful for very trivial derivations (such as `writeText` in Nixpkgs) that are cheaper to build than to substitute from a binary cache. + You may disable the effects of this attibute by enabling the + `always-allow-substitutes` configuration option in Nix. + > **Note** > > You need to have a builder configured which satisfies the diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 124165896..3d7a85047 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -25,7 +25,6 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -71,36 +70,10 @@ void emitTreeAttrs( v.mkAttrs(attrs); } -std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") -{ - state.checkURI(uri); - if (uri.find("://") == std::string::npos) { - const auto p = ParsedURL { - .scheme = defaultScheme, - .authority = "", - .path = uri - }; - return p.to_string(); - } else { - return uri; - } -} - -std::string fixURIForGit(std::string uri, EvalState & state) -{ - /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes - * them by removing the `:` and assuming a scheme of `ssh://` - * */ - static std::regex scp_uri("([^/]*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri)) - return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); - else - return fixURI(uri, state); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; + bool isFetchGit = false; }; static void fetchTree( @@ -108,11 +81,12 @@ static void fetchTree( const PosIdx pos, Value * * args, Value & v, - std::optional type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; NixStringContext context; + std::optional type; + if (params.isFetchGit) type = "git"; state.forceValue(*args[0], pos); @@ -142,10 +116,8 @@ static void fetchTree( if (attr.value->type() == nPath || attr.value->type() == nString) { auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace(state.symbols[attr.name], - state.symbols[attr.name] == "url" - ? type == "git" - ? fixURIForGit(s, state) - : fixURI(s, state) + params.isFetchGit && state.symbols[attr.name] == "url" + ? fixGitURL(s) : s); } else if (attr.value->type() == nBool) @@ -170,22 +142,24 @@ static void fetchTree( "while evaluating the first argument passed to the fetcher", false, false).toOwned(); - if (type == "git") { + if (params.isFetchGit) { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", fixURIForGit(url, state)); + attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(fixURI(url, state)); + input = fetchers::Input::fromURL(url); } } - if (!evalSettings.pureEval && !input.isDirect()) + if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; if (evalSettings.pureEval && !input.isLocked()) state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); + state.checkURI(input.toURLString()); + auto [tree, input2] = input.fetch(state.store); state.allowPath(tree.storePath); @@ -195,7 +169,7 @@ static void fetchTree( static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); + fetchTree(state, pos, args, v, { }); } static RegisterPrimOp primop_fetchTree({ @@ -203,12 +177,12 @@ static RegisterPrimOp primop_fetchTree({ .args = {"input"}, .doc = R"( Fetch a source tree or a plain file using one of the supported backends. - *input* can be an attribute set representation of [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) or a URL. - The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is allowed. + *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. + The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. Here are some examples of how to use `fetchTree`: - - Fetch a GitHub repository: + - Fetch a GitHub repository using the attribute set representation: ```nix builtins.fetchTree { @@ -219,7 +193,7 @@ static RegisterPrimOp primop_fetchTree({ } ``` - This evaluates to attribute set: + This evaluates to the following attribute set: ``` { @@ -231,10 +205,11 @@ static RegisterPrimOp primop_fetchTree({ shortRev = "ae2e6b3"; } ``` - - Fetch a single file from a URL: - ```nix - builtins.fetchTree "https://example.com/" + - Fetch the same GitHub repository using the URL-like syntax: + + ``` + builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" ``` )", .fun = prim_fetchTree, @@ -388,7 +363,12 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); + fetchTree(state, pos, args, v, + FetchTreeParams { + .emptyRevFallback = true, + .allowNameArgument = true, + .isFetchGit = true + }); } static RegisterPrimOp primop_fetchGit({ diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 17dcc18aa..57b010689 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -294,7 +294,6 @@ struct GitInputScheme : InputScheme if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") throw Error("unsupported Git input attribute '%s'", name); - parseURL(getStrAttr(attrs, "url")); maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); @@ -306,6 +305,9 @@ struct GitInputScheme : InputScheme Input input; input.attrs = attrs; + auto url = fixGitURL(getStrAttr(attrs, "url")); + parseURL(url); + input.attrs["url"] = url; return input; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index bd793c3d6..5ec332417 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -262,6 +262,14 @@ public: For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) )"}; + Setting alwaysAllowSubstitutes{ + this, false, "always-allow-substitutes", + R"( + If set to `true`, Nix will ignore the `allowSubstitutes` attribute in + derivations and always attempt to use available substituters. + For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md). + )"}; + Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7bf4476d0..703ded0b2 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,11 +3,10 @@ #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "common-protocol.hh" -#include "common-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -50,37 +49,31 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor bool good = true; /** - * Coercion to `CommonProto::ReadConn`. This makes it easy to use the - * factored out common protocol serialisers with a + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The common protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not common protocol - * serializers, once we have made that distiction. */ - operator CommonProto::ReadConn () + operator ServeProto::ReadConn () { - return CommonProto::ReadConn { + return ServeProto::ReadConn { .from = from, }; } /* - * Coercion to `CommonProto::WriteConn`. This makes it easy to use the - * factored out common protocol searlizers with a + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The common protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not common protocol - * serializers, once we have made that distiction. */ - operator CommonProto::WriteConn () + operator ServeProto::WriteConn () { - return CommonProto::WriteConn { + return ServeProto::WriteConn { .to = to, }; } @@ -183,7 +176,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = CommonProto::Serialise::read(*this, *conn); + info->references = ServeProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +210,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - CommonProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +239,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - CommonProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -331,7 +324,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = CommonProto::Serialise::read(*this, *conn); + auto builtOutputs = ServeProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -409,10 +402,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - CommonProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : CommonProto::Serialise::read(*this, *conn)) + for (auto & i : ServeProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +418,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - CommonProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - return CommonProto::Serialise::read(*this, *conn); + return ServeProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc4a94fab..1d900c272 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -122,7 +122,7 @@ bool ParsedDerivation::willBuildLocally(Store & localStore) const bool ParsedDerivation::substitutesAllowed() const { - return getBoolAttr("allowSubstitutes", true); + return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true); } bool ParsedDerivation::useUidRange() const diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh new file mode 100644 index 000000000..a3ce81026 --- /dev/null +++ b/src/libstore/serve-protocol-impl.hh @@ -0,0 +1,59 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "serve-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct ServeProto::Serialise +{ + static T read(const Store & store, ServeProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + +} diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc new file mode 100644 index 000000000..16a62b5bc --- /dev/null +++ b/src/libstore/serve-protocol.cc @@ -0,0 +1,15 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "archive.hh" + +#include + +namespace nix { + +/* protocol-specific definitions */ + +} diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 7e43b3969..e2345d450 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "common-protocol.hh" + namespace nix { #define SERVE_MAGIC_1 0x390c9deb @@ -10,6 +12,11 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + +class Store; +struct Source; + + /** * The "serve protocol", used by ssh:// stores. * @@ -22,6 +29,57 @@ struct ServeProto * Enumeration of all the request types for the protocol. */ enum struct Command : uint64_t; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + * + * This currently is just a `Source &`, but more fields will be added + * later. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + * + * This currently is just a `Sink &`, but more fields will be added + * later. + */ + struct WriteConn { + Sink & to; + }; + + /** + * Data type for canonical pairs of serialisers for the serve protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // See `worker-protocol.hh` for a longer explanation. +#if 0 + { + static T read(const Store & store, ReadConn conn); + static void write(const Store & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `ServeProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + ServeProto::Serialise::write(store, conn, t); + } }; enum struct ServeProto::Command : uint64_t @@ -58,4 +116,33 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) return s << (uint64_t) op; } +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define DECLARE_SERVE_SERIALISER(T) \ + struct ServeProto::Serialise< T > \ + { \ + static T read(const Store & store, ServeProto::ReadConn conn); \ + static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + }; + +template +DECLARE_SERVE_SERIALISER(std::vector); +template +DECLARE_SERVE_SERIALISER(std::set); +template +DECLARE_SERVE_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::map); +#undef COMMA_ + } diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc new file mode 100644 index 000000000..5d7df56cf --- /dev/null +++ b/src/libstore/tests/serve-protocol.cc @@ -0,0 +1,152 @@ +#include + +#include +#include + +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "serve-protocol"; + +using ServeProtoTest = ProtoTest; + +CHARACTERIZATION_TEST( + ServeProtoTest, + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 2970f8d2d..9b438e6cd 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -158,4 +158,21 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme) }; } +std::string fixGitURL(const std::string & url) +{ + std::regex scpRegex("([^/]*)@(.*):(.*)"); + if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) + return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); + else { + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); + } else + return url; + } +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d2413ec0e..26c2dcc28 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -45,4 +45,9 @@ struct ParsedUrlScheme { ParsedUrlScheme parseUrlScheme(std::string_view scheme); +/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + them by removing the `:` and assuming a scheme of `ssh://`. Also + changes absolute paths into file:// URLs. */ +std::string fixGitURL(const std::string & url); + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6fc22214a..9b6c80a75 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,10 +9,9 @@ #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "shared.hh" #include "util.hh" -#include "common-protocol.hh" -#include "common-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -821,8 +820,8 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); - CommonProto::ReadConn rconn { .from = in }; - CommonProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { .from = in }; + ServeProto::WriteConn wconn { .to = out }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +866,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = CommonProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,19 +875,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - CommonProto::write(*store, wconn, store->queryValidPaths(paths)); + ServeProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = CommonProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - CommonProto::write(*store, wconn, info->references); + ServeProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -916,7 +915,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(CommonProto::Serialise::read(*store, rconn), out); + store->exportPaths(ServeProto::Serialise::read(*store, rconn), out); break; } @@ -962,7 +961,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - CommonProto::write(*store, wconn, builtOutputs); + ServeProto::write(*store, wconn, builtOutputs); } break; @@ -971,9 +970,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(CommonProto::Serialise::read(*store, rconn), + store->computeFSClosure(ServeProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - CommonProto::write(*store, wconn, closure); + ServeProto::write(*store, wconn, closure); break; } @@ -988,7 +987,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = CommonProto::Serialise::read(*store, rconn); + info.references = ServeProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/src/nix/flake.md b/src/nix/flake.md index 939269c6a..f08648417 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -182,6 +182,12 @@ Currently the `type` attribute can be one of the following: git(+http|+https|+ssh|+git|+file|):(//)?(\?)? ``` + or + + ``` + @: + ``` + The `ref` attribute defaults to resolving the `HEAD` reference. The `rev` attribute must denote a commit that exists in the branch diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 418b4f63f..fc89f2040 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") [[ $path0 = $path0_ ]] +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 6de702d17..62ae8871b 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -186,6 +186,10 @@ in client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + # Test fetchTree on a github URL. + hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'") + assert hash == info['locked']['narHash'] + # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/unit-test-data/libstore/serve-protocol/content-address.bin new file mode 100644 index 000000000..8f14bcdb3 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/content-address.bin differ diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/unit-test-data/libstore/serve-protocol/drv-output.bin new file mode 100644 index 000000000..800a45fd8 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/drv-output.bin differ diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/unit-test-data/libstore/serve-protocol/optional-content-address.bin new file mode 100644 index 000000000..f8cfe65ba Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/optional-content-address.bin differ diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/unit-test-data/libstore/serve-protocol/optional-store-path.bin new file mode 100644 index 000000000..4fbca5576 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/optional-store-path.bin differ diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/unit-test-data/libstore/serve-protocol/realisation.bin new file mode 100644 index 000000000..2176c6c4a Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/realisation.bin differ diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/unit-test-data/libstore/serve-protocol/set.bin new file mode 100644 index 000000000..ce11ede7f Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/set.bin differ diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/unit-test-data/libstore/serve-protocol/store-path.bin new file mode 100644 index 000000000..3fc05f298 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/store-path.bin differ diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/unit-test-data/libstore/serve-protocol/string.bin new file mode 100644 index 000000000..aa7b5a604 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/string.bin differ diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/unit-test-data/libstore/serve-protocol/vector.bin new file mode 100644 index 000000000..7a37c8cd1 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/vector.bin differ