Merge remote-tracking branch 'nixos/master'

doc/manual/src/language/advanced-attributes.md

@@ -270,6 +270,9 @@ Derivations can declare some infrequently used optional attributes.
     useful for very trivial derivations (such as `writeText` in Nixpkgs)
     that are cheaper to build than to substitute from a binary cache.
 
+    You may disable the effects of this attibute by enabling the
+    `always-allow-substitutes` configuration option in Nix.
+
     > **Note**
     >
     > You need to have a builder configured which satisfies the

src/libexpr/primops/fetchTree.cc

@@ -25,7 +25,6 @@ void emitTreeAttrs(
 
     auto attrs = state.buildBindings(10);
 
-
     state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath));
 
     // FIXME: support arbitrary input attributes.
@@ -71,36 +70,10 @@ void emitTreeAttrs(
     v.mkAttrs(attrs);
 }
 
-std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
-{
-    state.checkURI(uri);
-    if (uri.find("://") == std::string::npos) {
-        const auto p = ParsedURL {
-            .scheme = defaultScheme,
-            .authority = "",
-            .path = uri
-        };
-        return p.to_string();
-    } else {
-        return uri;
-    }
-}
-
-std::string fixURIForGit(std::string uri, EvalState & state)
-{
-    /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
-     * them by removing the `:` and assuming a scheme of `ssh://`
-     * */
-    static std::regex scp_uri("([^/]*)@(.*):(.*)");
-    if (uri[0] != '/' && std::regex_match(uri, scp_uri))
-        return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
-    else
-        return fixURI(uri, state);
-}
-
 struct FetchTreeParams {
     bool emptyRevFallback = false;
     bool allowNameArgument = false;
+    bool isFetchGit = false;
 };
 
 static void fetchTree(
@@ -108,11 +81,12 @@ static void fetchTree(
     const PosIdx pos,
     Value * * args,
     Value & v,
-    std::optional<std::string> type,
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
     NixStringContext context;
+    std::optional<std::string> type;
+    if (params.isFetchGit) type = "git";
 
     state.forceValue(*args[0], pos);
 
@@ -142,10 +116,8 @@ static void fetchTree(
             if (attr.value->type() == nPath || attr.value->type() == nString) {
                 auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned();
                 attrs.emplace(state.symbols[attr.name],
-                    state.symbols[attr.name] == "url"
-                    ? type == "git"
-                      ? fixURIForGit(s, state)
-                      : fixURI(s, state)
+                    params.isFetchGit && state.symbols[attr.name] == "url"
+                    ? fixGitURL(s)
                     : s);
             }
             else if (attr.value->type() == nBool)
@@ -170,22 +142,24 @@ static void fetchTree(
                 "while evaluating the first argument passed to the fetcher",
                 false, false).toOwned();
 
-        if (type == "git") {
+        if (params.isFetchGit) {
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
-            attrs.emplace("url", fixURIForGit(url, state));
+            attrs.emplace("url", fixGitURL(url));
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
-            input = fetchers::Input::fromURL(fixURI(url, state));
+            input = fetchers::Input::fromURL(url);
         }
     }
 
-    if (!evalSettings.pureEval && !input.isDirect())
+    if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
         input = lookupInRegistries(state.store, input).first;
 
     if (evalSettings.pureEval && !input.isLocked())
         state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
 
+    state.checkURI(input.toURLString());
+
     auto [tree, input2] = input.fetch(state.store);
 
     state.allowPath(tree.storePath);
@@ -195,7 +169,7 @@ static void fetchTree(
 
 static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
+    fetchTree(state, pos, args, v, { });
 }
 
 static RegisterPrimOp primop_fetchTree({
@@ -203,12 +177,12 @@ static RegisterPrimOp primop_fetchTree({
     .args = {"input"},
     .doc = R"(
       Fetch a source tree or a plain file using one of the supported backends.
-      *input* can be an attribute set representation of [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) or a URL.
-      The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is allowed.
+      *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax.
+      The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled.
 
       Here are some examples of how to use `fetchTree`:
 
-      - Fetch a GitHub repository:
+      - Fetch a GitHub repository using the attribute set representation:
 
           ```nix
           builtins.fetchTree {
@@ -219,7 +193,7 @@ static RegisterPrimOp primop_fetchTree({
           }
           ```
 
-        This evaluates to attribute set:
+        This evaluates to the following attribute set:
 
           ```
           {
@@ -231,10 +205,11 @@ static RegisterPrimOp primop_fetchTree({
             shortRev = "ae2e6b3";
           }
           ```
-      - Fetch a single file from a URL:
 
-          ```nix
-          builtins.fetchTree "https://example.com/"
+      - Fetch the same GitHub repository using the URL-like syntax:
+
+          ```
+          builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
           ```
     )",
     .fun = prim_fetchTree,
@@ -388,7 +363,12 @@ static RegisterPrimOp primop_fetchTarball({
 
 static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
+    fetchTree(state, pos, args, v,
+        FetchTreeParams {
+            .emptyRevFallback = true,
+            .allowNameArgument = true,
+            .isFetchGit = true
+        });
 }
 
 static RegisterPrimOp primop_fetchGit({

src/libfetchers/git.cc

@@ -294,7 +294,6 @@ struct GitInputScheme : InputScheme
             if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
                 throw Error("unsupported Git input attribute '%s'", name);
 
-        parseURL(getStrAttr(attrs, "url"));
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
         maybeGetBoolAttr(attrs, "allRefs");
@@ -306,6 +305,9 @@ struct GitInputScheme : InputScheme
 
         Input input;
         input.attrs = attrs;
+        auto url = fixGitURL(getStrAttr(attrs, "url"));
+        parseURL(url);
+        input.attrs["url"] = url;
         return input;
     }
 

src/libstore/globals.hh

@@ -262,6 +262,14 @@ public:
           For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md)
         )"};
 
+    Setting<bool> alwaysAllowSubstitutes{
+        this, false, "always-allow-substitutes",
+        R"(
+          If set to `true`, Nix will ignore the `allowSubstitutes` attribute in
+          derivations and always attempt to use available substituters.
+          For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md).
+        )"};
+
     Setting<bool> buildersUseSubstitutes{
         this, false, "builders-use-substitutes",
         R"(

src/libstore/legacy-ssh-store.cc

@@ -3,11 +3,10 @@
 #include "pool.hh"
 #include "remote-store.hh"
 #include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 #include "build-result.hh"
 #include "store-api.hh"
 #include "path-with-outputs.hh"
-#include "common-protocol.hh"
-#include "common-protocol-impl.hh"
 #include "ssh.hh"
 #include "derivations.hh"
 #include "callback.hh"
@@ -50,37 +49,31 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         bool good = true;
 
         /**
-         * Coercion to `CommonProto::ReadConn`. This makes it easy to use the
-         * factored out common protocol serialisers with a
+         * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
+         * factored out serve protocol searlizers with a
          * `LegacySSHStore::Connection`.
          *
-         * The common protocol connection types are unidirectional, unlike
+         * The serve protocol connection types are unidirectional, unlike
          * this type.
-         *
-         * @todo Use server protocol serializers, not common protocol
-         * serializers, once we have made that distiction.
          */
-        operator CommonProto::ReadConn ()
+        operator ServeProto::ReadConn ()
         {
-            return CommonProto::ReadConn {
+            return ServeProto::ReadConn {
                 .from = from,
             };
         }
 
         /*
-         * Coercion to `CommonProto::WriteConn`. This makes it easy to use the
-         * factored out common protocol searlizers with a
+         * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
+         * factored out serve protocol searlizers with a
          * `LegacySSHStore::Connection`.
          *
-         * The common protocol connection types are unidirectional, unlike
+         * The serve protocol connection types are unidirectional, unlike
          * this type.
-         *
-         * @todo Use server protocol serializers, not common protocol
-         * serializers, once we have made that distiction.
          */
-        operator CommonProto::WriteConn ()
+        operator ServeProto::WriteConn ()
         {
-            return CommonProto::WriteConn {
+            return ServeProto::WriteConn {
                 .to = to,
             };
         }
@@ -183,7 +176,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = CommonProto::Serialise<StorePathSet>::read(*this, *conn);
+            info->references = ServeProto::Serialise<StorePathSet>::read(*this, *conn);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -217,7 +210,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            CommonProto::write(*this, *conn, info.references);
+            ServeProto::write(*this, *conn, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -246,7 +239,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            CommonProto::write(*this, *conn, info.references);
+            ServeProto::write(*this, *conn, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -331,7 +324,7 @@ public:
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
-            auto builtOutputs = CommonProto::Serialise<DrvOutputs>::read(*this, *conn);
+            auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(*this, *conn);
             for (auto && [output, realisation] : builtOutputs)
                 status.builtOutputs.insert_or_assign(
                     std::move(output.outputName),
@@ -409,10 +402,10 @@ public:
         conn->to
             << ServeProto::Command::QueryClosure
             << includeOutputs;
-        CommonProto::write(*this, *conn, paths);
+        ServeProto::write(*this, *conn, paths);
         conn->to.flush();
 
-        for (auto & i : CommonProto::Serialise<StorePathSet>::read(*this, *conn))
+        for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
             out.insert(i);
     }
 
@@ -425,10 +418,10 @@ public:
             << ServeProto::Command::QueryValidPaths
             << false // lock
             << maybeSubstitute;
-        CommonProto::write(*this, *conn, paths);
+        ServeProto::write(*this, *conn, paths);
         conn->to.flush();
 
-        return CommonProto::Serialise<StorePathSet>::read(*this, *conn);
+        return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
     }
 
     void connect() override

src/libstore/parsed-derivations.cc

@@ -122,7 +122,7 @@ bool ParsedDerivation::willBuildLocally(Store & localStore) const
 
 bool ParsedDerivation::substitutesAllowed() const
 {
-    return getBoolAttr("allowSubstitutes", true);
+    return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true);
 }
 
 bool ParsedDerivation::useUidRange() const

src/libstore/serve-protocol-impl.hh

@@ -0,0 +1,59 @@
+#pragma once
+/**
+ * @file
+ *
+ * Template implementations (as opposed to mere declarations).
+ *
+ * This file is an exmample of the "impl.hh" pattern. See the
+ * contributing guide.
+ */
+
+#include "serve-protocol.hh"
+#include "length-prefixed-protocol-helper.hh"
+
+namespace nix {
+
+/* protocol-agnostic templates */
+
+#define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
+    TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \
+    { \
+        return LengthPrefixedProtoHelper<ServeProto, T >::read(store, conn); \
+    } \
+    TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \
+    { \
+        LengthPrefixedProtoHelper<ServeProto, T >::write(store, conn, t); \
+    }
+
+SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
+SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::set<T>)
+SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
+
+#define COMMA_ ,
+SERVE_USE_LENGTH_PREFIX_SERIALISER(
+    template<typename K COMMA_ typename V>,
+    std::map<K COMMA_ V>)
+#undef COMMA_
+
+/**
+ * Use `CommonProto` where possible.
+ */
+template<typename T>
+struct ServeProto::Serialise
+{
+    static T read(const Store & store, ServeProto::ReadConn conn)
+    {
+        return CommonProto::Serialise<T>::read(store,
+            CommonProto::ReadConn { .from = conn.from });
+    }
+    static void write(const Store & store, ServeProto::WriteConn conn, const T & t)
+    {
+        CommonProto::Serialise<T>::write(store,
+            CommonProto::WriteConn { .to = conn.to },
+            t);
+    }
+};
+
+/* protocol-specific templates */
+
+}

src/libstore/serve-protocol.cc

@@ -0,0 +1,15 @@
+#include "serialise.hh"
+#include "util.hh"
+#include "path-with-outputs.hh"
+#include "store-api.hh"
+#include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
+#include "archive.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+/* protocol-specific definitions */
+
+}

src/libstore/serve-protocol.hh

@@ -1,6 +1,8 @@
 #pragma once
 ///@file
 
+#include "common-protocol.hh"
+
 namespace nix {
 
 #define SERVE_MAGIC_1 0x390c9deb
@@ -10,6 +12,11 @@ namespace nix {
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
+
+class Store;
+struct Source;
+
+
 /**
  * The "serve protocol", used by ssh:// stores.
  *
@@ -22,6 +29,57 @@ struct ServeProto
      * Enumeration of all the request types for the protocol.
      */
     enum struct Command : uint64_t;
+
+    /**
+     * A unidirectional read connection, to be used by the read half of the
+     * canonical serializers below.
+     *
+     * This currently is just a `Source &`, but more fields will be added
+     * later.
+     */
+    struct ReadConn {
+        Source & from;
+    };
+
+    /**
+     * A unidirectional write connection, to be used by the write half of the
+     * canonical serializers below.
+     *
+     * This currently is just a `Sink &`, but more fields will be added
+     * later.
+     */
+    struct WriteConn {
+        Sink & to;
+    };
+
+    /**
+     * Data type for canonical pairs of serialisers for the serve protocol.
+     *
+     * See https://en.cppreference.com/w/cpp/language/adl for the broader
+     * concept of what is going on here.
+     */
+    template<typename T>
+    struct Serialise;
+    // This is the definition of `Serialise` we *want* to put here, but
+    // do not do so.
+    //
+    // See `worker-protocol.hh` for a longer explanation.
+#if 0
+    {
+        static T read(const Store & store, ReadConn conn);
+        static void write(const Store & store, WriteConn conn, const T & t);
+    };
+#endif
+
+    /**
+     * Wrapper function around `ServeProto::Serialise<T>::write` that allows us to
+     * infer the type instead of having to write it down explicitly.
+     */
+    template<typename T>
+    static void write(const Store & store, WriteConn conn, const T & t)
+    {
+        ServeProto::Serialise<T>::write(store, conn, t);
+    }
 };
 
 enum struct ServeProto::Command : uint64_t
@@ -58,4 +116,33 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
     return s << (uint64_t) op;
 }
 
+/**
+ * Declare a canonical serialiser pair for the worker protocol.
+ *
+ * We specialise the struct merely to indicate that we are implementing
+ * the function for the given type.
+ *
+ * Some sort of `template<...>` must be used with the caller for this to
+ * be legal specialization syntax. See below for what that looks like in
+ * practice.
+ */
+#define DECLARE_SERVE_SERIALISER(T) \
+    struct ServeProto::Serialise< T > \
+    { \
+        static T read(const Store & store, ServeProto::ReadConn conn); \
+        static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \
+    };
+
+template<typename T>
+DECLARE_SERVE_SERIALISER(std::vector<T>);
+template<typename T>
+DECLARE_SERVE_SERIALISER(std::set<T>);
+template<typename... Ts>
+DECLARE_SERVE_SERIALISER(std::tuple<Ts...>);
+
+#define COMMA_ ,
+template<typename K, typename V>
+DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V>);
+#undef COMMA_
+
 }

src/libstore/tests/serve-protocol.cc

@@ -0,0 +1,152 @@
+#include <regex>
+
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+
+#include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
+#include "build-result.hh"
+#include "tests/protocol.hh"
+#include "tests/characterization.hh"
+
+namespace nix {
+
+const char commonProtoDir[] = "serve-protocol";
+
+using ServeProtoTest = ProtoTest<ServeProto, commonProtoDir>;
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    string,
+    "string",
+    (std::tuple<std::string, std::string, std::string, std::string, std::string> {
+        "",
+        "hi",
+        "white rabbit",
+        "大白兔",
+        "oh no \0\0\0 what was that!",
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    storePath,
+    "store-path",
+    (std::tuple<StorePath, StorePath> {
+        StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+        StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" },
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    contentAddress,
+    "content-address",
+    (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
+        ContentAddress {
+            .method = TextIngestionMethod {},
+            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+        },
+        ContentAddress {
+            .method = FileIngestionMethod::Flat,
+            .hash = hashString(HashType::htSHA1, "blob blob..."),
+        },
+        ContentAddress {
+            .method = FileIngestionMethod::Recursive,
+            .hash = hashString(HashType::htSHA256, "(...)"),
+        },
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    drvOutput,
+    "drv-output",
+    (std::tuple<DrvOutput, DrvOutput> {
+        {
+            .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+            .outputName = "baz",
+        },
+        DrvOutput {
+            .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
+            .outputName = "quux",
+        },
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    realisation,
+    "realisation",
+    (std::tuple<Realisation, Realisation> {
+        Realisation {
+            .id = DrvOutput {
+                .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+                .outputName = "baz",
+            },
+            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+            .signatures = { "asdf", "qwer" },
+        },
+        Realisation {
+            .id = {
+                .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+                .outputName = "baz",
+            },
+            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+            .signatures = { "asdf", "qwer" },
+            .dependentRealisations = {
+                {
+                    DrvOutput {
+                        .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
+                        .outputName = "quux",
+                    },
+                    StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+                },
+            },
+        },
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    vector,
+    "vector",
+    (std::tuple<std::vector<std::string>, std::vector<std::string>, std::vector<std::string>, std::vector<std::vector<std::string>>> {
+        { },
+        { "" },
+        { "", "foo", "bar" },
+        { {}, { "" }, { "", "1", "2" } },
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    set,
+    "set",
+    (std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
+        { },
+        { "" },
+        { "", "foo", "bar" },
+        { {}, { "" }, { "", "1", "2" } },
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    optionalStorePath,
+    "optional-store-path",
+    (std::tuple<std::optional<StorePath>, std::optional<StorePath>> {
+        std::nullopt,
+        std::optional {
+            StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" },
+        },
+    }))
+
+CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    optionalContentAddress,
+    "optional-content-address",
+    (std::tuple<std::optional<ContentAddress>, std::optional<ContentAddress>> {
+        std::nullopt,
+        std::optional {
+            ContentAddress {
+                .method = FileIngestionMethod::Flat,
+                .hash = hashString(HashType::htSHA1, "blob blob..."),
+            },
+        },
+    }))
+
+}

src/libutil/url.cc

@@ -158,4 +158,21 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme)
     };
 }
 
+std::string fixGitURL(const std::string & url)
+{
+    std::regex scpRegex("([^/]*)@(.*):(.*)");
+    if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex))
+        return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3");
+    else {
+        if (url.find("://") == std::string::npos) {
+            return (ParsedURL {
+                .scheme = "file",
+                .authority = "",
+                .path = url
+            }).to_string();
+        } else
+            return url;
+    }
+}
+
 }

src/libutil/url.hh

@@ -45,4 +45,9 @@ struct ParsedUrlScheme {
 
 ParsedUrlScheme parseUrlScheme(std::string_view scheme);
 
+/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
+   them by removing the `:` and assuming a scheme of `ssh://`. Also
+   changes absolute paths into file:// URLs. */
+std::string fixGitURL(const std::string & url);
+
 }

src/nix-store/nix-store.cc

@@ -9,10 +9,9 @@
 #include "local-store.hh"
 #include "monitor-fd.hh"
 #include "serve-protocol.hh"
+#include "serve-protocol-impl.hh"
 #include "shared.hh"
 #include "util.hh"
-#include "common-protocol.hh"
-#include "common-protocol-impl.hh"
 #include "graphml.hh"
 #include "legacy.hh"
 #include "path-with-outputs.hh"
@@ -821,8 +820,8 @@ static void opServe(Strings opFlags, Strings opArgs)
     out.flush();
     unsigned int clientVersion = readInt(in);
 
-    CommonProto::ReadConn rconn { .from = in };
-    CommonProto::WriteConn wconn { .to = out };
+    ServeProto::ReadConn rconn { .from = in };
+    ServeProto::WriteConn wconn { .to = out };
 
     auto getBuildSettings = [&]() {
         // FIXME: changing options here doesn't work if we're
@@ -867,7 +866,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case ServeProto::Command::QueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -876,19 +875,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                     store->substitutePaths(paths);
                 }
 
-                CommonProto::write(*store, wconn, store->queryValidPaths(paths));
+                ServeProto::write(*store, wconn, store->queryValidPaths(paths));
                 break;
             }
 
             case ServeProto::Command::QueryPathInfos: {
-                auto paths = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
+                auto paths = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        CommonProto::write(*store, wconn, info->references);
+                        ServeProto::write(*store, wconn, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -916,7 +915,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case ServeProto::Command::ExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(CommonProto::Serialise<StorePathSet>::read(*store, rconn), out);
+                store->exportPaths(ServeProto::Serialise<StorePathSet>::read(*store, rconn), out);
                 break;
             }
 
@@ -962,7 +961,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                     DrvOutputs builtOutputs;
                     for (auto & [output, realisation] : status.builtOutputs)
                         builtOutputs.insert_or_assign(realisation.id, realisation);
-                    CommonProto::write(*store, wconn, builtOutputs);
+                    ServeProto::write(*store, wconn, builtOutputs);
                 }
 
                 break;
@@ -971,9 +970,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case ServeProto::Command::QueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(CommonProto::Serialise<StorePathSet>::read(*store, rconn),
+                store->computeFSClosure(ServeProto::Serialise<StorePathSet>::read(*store, rconn),
                     closure, false, includeOutputs);
-                CommonProto::write(*store, wconn, closure);
+                ServeProto::write(*store, wconn, closure);
                 break;
             }
 
@@ -988,7 +987,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
+                info.references = ServeProto::Serialise<StorePathSet>::read(*store, rconn);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));

src/nix/flake.md

@@ -182,6 +182,12 @@ Currently the `type` attribute can be one of the following:
   git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)?
   ```
 
+  or
+
+  ```
+  <user>@<server>:<path>
+  ```
+
   The `ref` attribute defaults to resolving the `HEAD` reference.
 
   The `rev` attribute must denote a commit that exists in the branch

tests/functional/fetchGit.sh

@@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP
 path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
 path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
 [[ $path0 = $path0_ ]]
+path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath")
+[[ $path0 = $path0_ ]]
 export _NIX_FORCE_HTTP=1
 [[ $(tail -n 1 $path0/hello) = "hello" ]]
 

tests/nixos/github-flakes.nix

@@ -186,6 +186,10 @@ in
     client.succeed("nix registry pin nixpkgs")
     client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2")
 
+    # Test fetchTree on a github URL.
+    hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'")
+    assert hash == info['locked']['narHash']
+
     # Shut down the web server. The flake should be cached on the client.
     github.succeed("systemctl stop httpd.service")