Merge pull request #5342 from Misterio77/add-sourcehut

Add support for sourcehut input scheme

flake.nix

@@ -501,6 +501,12 @@
           inherit (self) overlay;
         });
 
+        tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
         tests.setuid = nixpkgs.lib.genAttrs
           ["i686-linux" "x86_64-linux"]
           (system:

src/libfetchers/github.cc

@@ -8,6 +8,7 @@
 
 #include <optional>
 #include <nlohmann/json.hpp>
+#include <fstream>
 
 namespace nix::fetchers {
 
@@ -17,7 +18,7 @@ struct DownloadUrl
     Headers headers;
 };
 
-// A github or gitlab host
+// A github, gitlab, or sourcehut host
 const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
 std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
@@ -345,7 +346,95 @@ struct GitLabInputScheme : GitArchiveInputScheme
     }
 };
 
+struct SourceHutInputScheme : GitArchiveInputScheme
+{
+    std::string type() override { return "sourcehut"; }
+
+    std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
+    {
+        // SourceHut supports both PAT and OAuth2. See
+        // https://man.sr.ht/meta.sr.ht/oauth.md
+        return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
+        // Note: This currently serves no purpose, as this kind of authorization
+        // does not allow for downloading tarballs on sourcehut private repos.
+        // Once it is implemented, however, should work as expected.
+    }
+
+    Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
+    {
+        // TODO: In the future, when the sourcehut graphql API is implemented for mercurial
+        // and with anonymous access, this method should use it instead.
+
+        auto ref = *input.getRef();
+
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
+        auto base_url = fmt("https://%s/%s/%s",
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
+
+        Headers headers = makeHeadersWithAuthTokens(host);
+
+        std::string ref_uri;
+        if (ref == "HEAD") {
+            auto file = store->toRealPath(
+                downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath);
+            std::ifstream is(file);
+            std::string line;
+            getline(is, line);
+
+            auto ref_index = line.find("ref: ");
+            if (ref_index == std::string::npos) {
+                throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref);
+            }
+
+            ref_uri = line.substr(ref_index+5, line.length()-1);
+        } else
+            ref_uri = fmt("refs/heads/%s", ref);
+
+        auto file = store->toRealPath(
+            downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
+        std::ifstream is(file);
+
+        std::string line;
+        std::string id;
+        while(getline(is, line)) {
+            auto index = line.find(ref_uri);
+            if (index != std::string::npos) {
+                id = line.substr(0, index-1);
+                break;
+            }
+        }
+
+        if(id.empty())
+            throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
+
+        auto rev = Hash::parseAny(id, htSHA1);
+        debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
+        return rev;
+    }
+
+    DownloadUrl getDownloadUrl(const Input & input) const override
+    {
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
+        auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz",
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            input.getRev()->to_string(Base16, false));
+
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl { url, headers };
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
+        Input::fromURL(fmt("git+https://%s/%s/%s",
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+            .applyOverrides(input.getRef(), input.getRev())
+            .clone(destDir);
+    }
+};
+
 static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
 static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
+static auto rSourceHutInputScheme = OnStartup([] { registerInputScheme(std::make_unique<SourceHutInputScheme>()); });
 
 }

src/nix/flake.md

@@ -209,6 +209,38 @@ Currently the `type` attribute can be one of the following:
   * `github:edolstra/dwarffs/unstable`
   * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31`
 
+* `sourcehut`: Similar to `github`, is a more efficient way to fetch
+  SourceHut repositories. The following attributes are required:
+
+  * `owner`: The owner of the repository (including leading `~`).
+
+  * `repo`: The name of the repository.
+
+  Like `github`, these are downloaded as tarball archives.
+
+  The URL syntax for `sourcehut` flakes is:
+
+  `sourcehut:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)?`
+
+  `<rev-or-ref>` works the same as `github`. Either a branch or tag name
+  (`ref`), or a commit hash (`rev`) can be specified.
+
+  Since SourceHut allows for self-hosting, you can specify `host` as
+  a parameter, to point to any instances other than `git.sr.ht`.
+
+  Currently, `ref` name resolution only works for Git repositories.
+  You can refer to Mercurial repositories by simply changing `host` to
+  `hg.sr.ht` (or any other Mercurial instance). With the caveat
+  that you must explicitly specify a commit hash (`rev`).
+
+  Some examples:
+
+  * `sourcehut:~misterio/nix-colors`
+  * `sourcehut:~misterio/nix-colors/main`
+  * `sourcehut:~misterio/nix-colors?host=git.example.org`
+  * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c`
+  * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht`
+
 * `indirect`: Indirections through the flake registry. These have the
   form
 

tests/sourcehut-flakes.nix

@@ -0,0 +1,156 @@
+{ nixpkgs, system, overlay }:
+
+with import (nixpkgs + "/nixos/lib/testing-python.nix")
+{
+  inherit system;
+  extraConfigurations = [{ nixpkgs.overlays = [ overlay ]; }];
+};
+
+let
+  # Generate a fake root CA and a fake git.sr.ht certificate.
+  cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; }
+    ''
+      mkdir -p $out
+
+      openssl genrsa -out ca.key 2048
+      openssl req -new -x509 -days 36500 -key ca.key \
+        -subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt
+
+      openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \
+        -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=git.sr.ht" -out server.csr
+      openssl x509 -req -extfile <(printf "subjectAltName=DNS:git.sr.ht") \
+        -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt
+    '';
+
+  registry = pkgs.writeTextFile {
+    name = "registry";
+    text = ''
+      {
+        "flakes": [
+          {
+            "from": {
+              "type": "indirect",
+              "id": "nixpkgs"
+            },
+            "to": {
+              "type": "sourcehut",
+              "owner": "~NixOS",
+              "repo": "nixpkgs"
+            }
+          }
+        ],
+        "version": 2
+      }
+    '';
+    destination = "/flake-registry.json";
+  };
+
+  nixpkgs-repo = pkgs.runCommand "nixpkgs-flake" { }
+    ''
+      dir=NixOS-nixpkgs-${nixpkgs.shortRev}
+      cp -prd ${nixpkgs} $dir
+
+      # Set the correct timestamp in the tarball.
+      find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} --
+
+      mkdir -p $out/archive
+      tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference
+
+      echo 'ref: refs/heads/master' > $out/HEAD
+
+      mkdir -p $out/info
+      echo '${nixpkgs.rev} refs/heads/master' > $out/info/refs
+    '';
+
+in
+
+makeTest (
+
+  {
+    name = "sourcehut-flakes";
+
+    nodes =
+      {
+        # Impersonate git.sr.ht
+        sourcehut =
+          { config, pkgs, ... }:
+          {
+            networking.firewall.allowedTCPPorts = [ 80 443 ];
+
+            services.httpd.enable = true;
+            services.httpd.adminAddr = "foo@example.org";
+            services.httpd.extraConfig = ''
+              ErrorLog syslog:local6
+            '';
+            services.httpd.virtualHosts."git.sr.ht" =
+              {
+                forceSSL = true;
+                sslServerKey = "${cert}/server.key";
+                sslServerCert = "${cert}/server.crt";
+                servedDirs =
+                  [
+                    {
+                      urlPath = "/~NixOS/nixpkgs";
+                      dir = nixpkgs-repo;
+                    }
+                    {
+                      urlPath = "/~NixOS/flake-registry/blob/master";
+                      dir = registry;
+                    }
+                  ];
+              };
+          };
+
+        client =
+          { config, lib, pkgs, nodes, ... }:
+          {
+            virtualisation.writableStore = true;
+            virtualisation.diskSize = 2048;
+            virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ];
+            virtualisation.memorySize = 4096;
+            nix.binaryCaches = lib.mkForce [ ];
+            nix.extraOptions = ''
+              experimental-features = nix-command flakes
+              flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json
+            '';
+            environment.systemPackages = [ pkgs.jq ];
+            networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} =
+              [ "git.sr.ht" ];
+            security.pki.certificateFiles = [ "${cert}/ca.crt" ];
+          };
+      };
+
+    testScript = { nodes }: ''
+      # fmt: off
+      import json
+      import time
+
+      start_all()
+
+      sourcehut.wait_for_unit("httpd.service")
+
+      client.succeed("curl -v https://git.sr.ht/ >&2")
+      client.succeed("nix registry list | grep nixpkgs")
+
+      rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision")
+      assert rev.strip() == "${nixpkgs.rev}", "revision mismatch"
+
+      client.succeed("nix registry pin nixpkgs")
+
+      client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2")
+
+      # Shut down the web server. The flake should be cached on the client.
+      sourcehut.succeed("systemctl stop httpd.service")
+
+      info = json.loads(client.succeed("nix flake info nixpkgs --json"))
+      date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified']))
+      assert date == "${nixpkgs.lastModifiedDate}", "time mismatch"
+
+      client.succeed("nix build nixpkgs#hello")
+
+      # The build shouldn't fail even with --tarball-ttl 0 (the server
+      # being down should not be a fatal error).
+      client.succeed("nix build nixpkgs#fuse --tarball-ttl 0")
+    '';
+
+  })