Merge pull request #9676 from DavHau/git-testsuite

initialize test suite for git fetchers

tests/nixos/default.nix

@@ -5,14 +5,28 @@ let
   nixos-lib = import (nixpkgs + "/nixos/lib") { };
 
   # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
-  runNixOSTestFor = system: test: nixos-lib.runTest {
+  runNixOSTestFor = system: test:
-    imports = [ test ];
+    (nixos-lib.runTest {
-    hostPkgs = nixpkgsFor.${system}.native;
+      imports = [ test ];
-    defaults = {
+      hostPkgs = nixpkgsFor.${system}.native;
-      nixpkgs.pkgs = nixpkgsFor.${system}.native;
+      defaults = {
+        nixpkgs.pkgs = nixpkgsFor.${system}.native;
+        nix.checkAllErrors = false;
+      };
+      _module.args.nixpkgs = nixpkgs;
+      _module.args.system = system;
+    })
+    // {
+      # allow running tests against older nix versions via `nix eval --apply`
+      # Example:
+      #   nix build "$(nix eval --raw --impure .#hydraJobs.tests.fetch-git --apply 't: (t.forNix "2.19.2").drvPath')^*"
+      forNix = nixVersion: runNixOSTestFor system {
+        imports = [test];
+        defaults.nixpkgs.overlays = [(curr: prev: {
+          nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix;
+        })];
+      };
     };
-    _module.args.nixpkgs = nixpkgs;
-  };
 
 in
 
@@ -40,4 +54,6 @@ in
   setuid = lib.genAttrs
     ["i686-linux" "x86_64-linux"]
     (system: runNixOSTestFor system ./setuid.nix);
+
+  fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git;
 }

tests/nixos/fetch-git/default.nix

@@ -0,0 +1,32 @@
+{ lib, config, ... }:
+{
+  name = "fetch-git";
+
+  imports = [
+    ./testsupport/gitea.nix
+  ];
+
+  /*
+  Test cases
+
+  Test cases are automatically imported from ./test-cases/{name}
+
+  The following is set up automatically for each test case:
+    - a repo with the {name} is created on the gitea server
+    - a repo with the {name} is created on the client
+    - the client repo is configured to push to the server repo
+
+  Python variables:
+    - repo.path: the path to the directory of the client repo
+    - repo.git: the git command with the client repo as the working directory
+    - repo.remote: the url to the server repo
+  */
+  testCases =
+    map
+    (testCaseName: {...}: {
+      imports = [ (./test-cases + "/${testCaseName}") ];
+      # ensures tests are named like their directories they are defined in
+      name = testCaseName;
+    })
+    (lib.attrNames (builtins.readDir ./test-cases));
+}

tests/nixos/fetch-git/test-cases/http-simple/default.nix

@@ -0,0 +1,39 @@
+{ config, ... }:
+{
+  description = "can fetch a git repo via http";
+  script = ''
+    # add a file to the repo
+    client.succeed(f"""
+      echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \
+      && echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add test-case thailand \
+      && {repo.git} commit -m 'commit1'
+    """)
+
+    # save the revision
+    rev1 = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # push to the server
+    client.succeed(f"""
+      {repo.git} push origin main
+    """)
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath"
+    """)
+
+    # check if the committed file is there
+    client.succeed(f"""
+      test -f {fetched1}/thailand
+    """)
+
+    # check if the revision is the same
+    rev1_fetched = client.succeed(f"""
+      nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev"
+    """).strip()
+    assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
+  '';
+}

tests/nixos/fetch-git/test-cases/ssh-simple/default.nix

@@ -0,0 +1,43 @@
+{ config, ... }:
+{
+  description = "can fetch a git repo via ssh";
+  script = ''
+    # add a file to the repo
+    client.succeed(f"""
+      echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \
+      && echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add test-case thailand \
+      && {repo.git} commit -m 'commit1'
+    """)
+
+    # save the revision
+    rev1 = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # push to the server
+    client.succeed(f"""
+      {repo.git} push origin-ssh main
+    """)
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr '
+        (builtins.fetchGit "{repo.remote_ssh}").outPath
+      '
+    """)
+
+    # check if the committed file is there
+    client.succeed(f"""
+      test -f {fetched1}/thailand
+    """)
+
+    # check if the revision is the same
+    rev1_fetched = client.succeed(f"""
+      nix eval --impure --raw --expr '
+        (builtins.fetchGit "{repo.remote_ssh}").rev
+      '
+    """).strip()
+    assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
+  '';
+}

tests/nixos/fetch-git/testsupport/gitea-repo.nix

@@ -0,0 +1,51 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption types;
+
+  testCaseExtension = { config, ... }: {
+    setupScript = ''
+      repo = Repo("${config.name}")
+    '';
+  };
+in
+{
+  options = {
+    testCases = mkOption {
+      type = types.listOf (types.submodule testCaseExtension);
+    };
+  };
+  config = {
+    setupScript = ''
+      class Repo:
+        """
+        A class to create a git repository on the gitea server and locally.
+        """
+        def __init__(self, name):
+          self.name = name
+          self.path = "/tmp/repos/" + name
+          self.remote = "http://gitea:3000/test/" + name
+          self.remote_ssh = "ssh://gitea/root/" + name
+          self.git = f"git -C {self.path}"
+          self.create()
+
+        def create(self):
+          # create ssh remote repo
+          gitea.succeed(f"""
+            git init --bare -b main /root/{self.name}
+          """)
+          # create http remote repo
+          gitea.succeed(f"""
+            curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \
+              -H 'Accept: application/json' -H 'Content-Type: application/json' \
+              -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )}
+          """)
+          # setup git remotes on client
+          client.succeed(f"""
+            mkdir -p {self.path} \
+            && git init -b main {self.path} \
+            && {self.git} remote add origin {self.remote} \
+            && {self.git} remote add origin-ssh root@gitea:{self.name}
+          """)
+    '';
+  };
+}

tests/nixos/fetch-git/testsupport/gitea.nix

@@ -0,0 +1,102 @@
+{ lib, nixpkgs, system, pkgs, ... }: let
+  clientPrivateKey = pkgs.writeText "id_ed25519" ''
+    -----BEGIN OPENSSH PRIVATE KEY-----
+    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+    QyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQAAAJAwVQ5VMFUO
+    VQAAAAtzc2gtZWQyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQ
+    AAAEB7lbfkkdkJoE+4TKHPdPQWBKLSx+J54Eg8DaTr+3KoSlt5a8eH8BYZYjoQhzXGVKKH
+    Je1pw1D0p7O2Vb9VTLzBAAAACGJmb0BtaW5pAQIDBAU=
+    -----END OPENSSH PRIVATE KEY-----
+  '';
+
+  clientPublicKey =
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFt5a8eH8BYZYjoQhzXGVKKHJe1pw1D0p7O2Vb9VTLzB";
+
+in {
+  imports = [
+    ../testsupport/setup.nix
+    ../testsupport/gitea-repo.nix
+  ];
+  nodes = {
+    gitea = { pkgs, ... }: {
+      services.gitea.enable = true;
+      services.gitea.settings.service.DISABLE_REGISTRATION = true;
+      services.gitea.settings.log.LEVEL = "Info";
+      services.gitea.settings.database.LOG_SQL = false;
+      services.openssh.enable = true;
+      networking.firewall.allowedTCPPorts = [ 3000 ];
+      environment.systemPackages = [ pkgs.git pkgs.gitea ];
+
+      users.users.root.openssh.authorizedKeys.keys = [clientPublicKey];
+
+      # TODO: remove this after updating to nixos-23.11
+      nixpkgs.pkgs = lib.mkForce (import nixpkgs {
+        inherit system;
+        config.permittedInsecurePackages = [
+          "gitea-1.19.4"
+        ];
+      });
+    };
+    client = { pkgs, ... }: {
+      environment.systemPackages = [ pkgs.git ];
+    };
+  };
+  defaults = { pkgs, ... }: {
+    environment.systemPackages = [ pkgs.jq ];
+  };
+
+  setupScript = ''
+    import shlex
+
+    gitea.wait_for_unit("gitea.service")
+
+    gitea_admin = "test"
+    gitea_admin_password = "test123test"
+
+    gitea.succeed(f"""
+      gitea --version >&2
+      su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create \
+        --username {gitea_admin} --password {gitea_admin_password} --email test@client'
+    """)
+
+    client.wait_for_unit("multi-user.target")
+    gitea.wait_for_open_port(3000)
+
+    gitea_admin_token = gitea.succeed(f"""
+      curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/users/test/tokens \
+        -H 'Accept: application/json' -H 'Content-Type: application/json' \
+        -d {shlex.quote( '{"name":"token", "scopes":["all"]}' )} \
+        | jq -r '.sha1'
+    """).strip()
+
+    client.succeed(f"""
+      echo "http://{gitea_admin}:{gitea_admin_password}@gitea:3000" >~/.git-credentials-admin
+      git config --global credential.helper 'store --file ~/.git-credentials-admin'
+      git config --global user.email "test@client"
+      git config --global user.name "Test User"
+      git config --global gc.autodetach 0
+      git config --global gc.auto 0
+    """)
+
+    # add client's private key to ~/.ssh
+    client.succeed("""
+      mkdir -p ~/.ssh
+      chmod 700 ~/.ssh
+      cat ${clientPrivateKey} >~/.ssh/id_ed25519
+      chmod 600 ~/.ssh/id_ed25519
+    """)
+
+    client.succeed("""
+      echo "Host gitea" >>~/.ssh/config
+      echo "  StrictHostKeyChecking no" >>~/.ssh/config
+      echo "  UserKnownHostsFile /dev/null" >>~/.ssh/config
+      echo "  User root" >>~/.ssh/config
+    """)
+
+    # ensure ssh from client to gitea works
+    client.succeed("""
+      ssh root@gitea true
+    """)
+
+  '';
+}

tests/nixos/fetch-git/testsupport/setup.nix

@@ -0,0 +1,91 @@
+{ lib, config, extendModules, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+
+  indent = lib.replaceStrings ["\n"] ["\n    "];
+
+  execTestCase = testCase: ''
+
+    ### TEST ${testCase.name}: ${testCase.description} ###
+
+    with subtest("${testCase.description}"):
+        # Setup
+        ${indent testCase.setupScript}
+
+        # Test
+        ${indent testCase.script}
+  '';
+in
+{
+
+  options = {
+    setupScript = mkOption {
+      type = types.lines;
+      description = ''
+        Python code that runs before the main test.
+
+        Variables defined by this code will be available in the test.
+      '';
+      default = "";
+    };
+    testCases = mkOption {
+      description = ''
+        The test cases. See `testScript`.
+      '';
+      type = types.listOf (types.submodule {
+        options.name = mkOption {
+          type = types.str;
+          description = ''
+            The name of the test case.
+
+            A repository with that name will be set up on the gitea server and locally.
+          '';
+        };
+        options.description = mkOption {
+          type = types.str;
+          description = ''
+            A description of the test case.
+          '';
+        };
+        options.setupScript = mkOption {
+          type = types.lines;
+          description = ''
+            Python code that runs before the test case.
+          '';
+          default = "";
+        };
+        options.script = mkOption {
+          type = types.lines;
+          description = ''
+            Python code that runs the test.
+
+            Variables defined by the global `setupScript`, as well as `testCases.*.setupScript` will be available here.
+          '';
+        };
+      });
+    };
+  };
+
+  config = {
+    nodes.client = {
+      environment.variables = {
+        _NIX_FORCE_HTTP = "1";
+      };
+      nix.settings.experimental-features = ["nix-command" "flakes"];
+    };
+    setupScript = ''
+    '';
+    testScript = ''
+      start_all();
+
+      ${config.setupScript}
+
+      ### SETUP COMPLETE ###
+
+      ${lib.concatStringsSep "\n" (map execTestCase config.testCases)}
+    '';
+  };
+}