cluster/simulacrum: only run checks on x86_64-linux

`flake.lock`: Update

cluster/catalog/secrets.nix

@@ -46,6 +46,7 @@ in
               };
             }) // (if secretConfig.shared then let
               secretFile = "${svcName}-${secretName}.age";
+              snakeoilFile = "${svcName}-${secretName}-snakeoil.txt";
             in {
               editSecret = {
                 description = "Edit this secret";
@@ -54,15 +55,31 @@ in
                   agenix -e '${secretFile}'
                 '';
               };
-            } else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
-              secretFile = "${svcName}-${secretName}-${node}.age";
-            in {
-              description = "Edit this secret for '${node}'";
-              command = ''
-                ${setupCommands secretFile [ node ]}
-                agenix -e '${secretFile}'
-              '';
-            })));
+              editSnakeoil = {
+                description = "Edit this secret's snakeoil";
+                command = ''
+                  $EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
+                '';
+              };
+            } else lib.mkMerge [
+              (lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
+                secretFile = "${svcName}-${secretName}-${node}.age";
+              in {
+                description = "Edit this secret for '${node}'";
+                command = ''
+                  ${setupCommands secretFile [ node ]}
+                  agenix -e '${secretFile}'
+                '';
+              })))
+              (lib.mapAttrs' (name: lib.nameValuePair "editSnakeoilInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
+                snakeoilFile = "${svcName}-${secretName}-${node}-snakeoil.txt";
+              in {
+                description = "Edit this secret's snakeoil for '${node}'";
+                command = ''
+                  $EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
+                '';
+              })))
+            ]);
           };
         }) svcConfig.secrets))
         lib.concatLists

cluster/default.nix

@@ -16,6 +16,8 @@ lib.evalModules {
     ./lib/port-magic-multi.nix
     ./lib/mesh.nix
     ./lib/secrets.nix
+    ./lib/testing.nix
+    ./lib/lib.nix
 
     ./import-services.nix
   ];

cluster/lib/inject-nixos-config.nix

@@ -2,9 +2,9 @@
 with lib;
 
 {
-  options.out.injectNixosConfig = mkOption {
-    description = "NixOS configuration to inject into the given host.";
-    type = with types; functionTo raw;
+  options.out = mkOption {
+    description = "Output functions.";
+    type = with types; lazyAttrsOf (functionTo raw);
     default = const [];
   };
 }

cluster/lib/lib.nix

@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+
+{
+  options.lib = {
+    forService = lib.mkOption {
+      description = "Enable these definitions for a particular service only.";
+      type = lib.types.functionTo lib.types.raw;
+      readOnly = true;
+      default = service: lib.mkIf (!config.simulacrum || lib.any (s: s == service) config.testConfig.activeServices);
+    };
+  };
+}

cluster/lib/service-module.nix

@@ -52,6 +52,24 @@ in
       }));
       default = {};
     };
+    simulacrum = {
+      enable = mkEnableOption "testing this service in the Simulacrum";
+      deps = mkOption {
+        description = "Other services to include.";
+        type = with types; listOf str;
+        default = [];
+      };
+      settings = mkOption {
+        description = "NixOS test configuration.";
+        type = types.deferredModule;
+        default = {};
+      };
+      augments = mkOption {
+        description = "Cluster augments (will be propagated).";
+        type = types.deferredModule;
+        default = {};
+      };
+    };
   };
   config.otherNodes = builtins.mapAttrs (const filterGroup) config.nodes;
 }

cluster/lib/services.nix

@@ -39,7 +39,11 @@ in
     default = {};
   };
 
-  config.out.injectNixosConfig = hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) config.services)) ++ [
-    introspectionModule
-  ];
+  config.out = {
+    injectNixosConfigForServices = services: hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) (lib.getAttrs services config.services))) ++ [
+      introspectionModule
+    ];
+
+    injectNixosConfig = config.out.injectNixosConfigForServices (lib.attrNames config.services);
+  };
 }

cluster/lib/testing.nix

@@ -0,0 +1,15 @@
+{ lib, ... }:
+
+{
+  options = {
+    simulacrum = lib.mkOption {
+      description = "Whether we are in the Simulacrum.";
+      type = lib.types.bool;
+      default = false;
+    };
+    testConfig = lib.mkOption {
+      type = lib.types.attrs;
+      readOnly = true;
+    };
+  };
+}

cluster/part.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ./catalog
+    ./simulacrum/checks.nix
   ];
 
   options.cluster = lib.mkOption {

cluster/services/attic/default.nix

@@ -25,7 +25,7 @@
     };
   };
 
-  garage = {
+  garage = config.lib.forService "attic" {
     keys.attic.locksmith = {
       nodes = config.services.attic.nodes.server;
       owner = "atticd";
@@ -40,7 +40,7 @@
     serverAddrs = map
       (node: depot.hours.${node}.interfaces.primary.addrPublic)
       config.services.attic.nodes.server;
-  in {
+  in config.lib.forService "attic" {
     cache-api.target = serverAddrs;
     cache.target = serverAddrs;
   };

cluster/services/c-f32aebf5/default.nix

@@ -1,6 +0,0 @@
-{
-  services.c-f32aebf5 = {
-    nodes.host = [ "VEGAS" ];
-    nixos.host = [ ./host.nix ];
-  };
-}

cluster/services/c-f32aebf5/host.nix

@@ -1,203 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  cid = "c-f32aebf5";
-  link = config.links.${cid};
-  root = "/var/lib/${cid}";
-  home = "${root}/pfx";
-
-  sptAki =  {
-    release-3_8_0 = pkgs.fetchurl {
-      url = "https://dev.sp-tarkov.com/SPT/Stable-releases/releases/download/3.8.0/RELEASE-SPT-3.8.0-29197-2dd4d91.7z";
-      hash = "sha256-IRMzI+hQkoCmVJXkAV4c/b2l/MtLb98IwDftMbFTlxA=";
-    };
-    update-3_8_1 = pkgs.fetchurl {
-      url = "https://spt-releases.modd.in/SPT-3.8.1-29197-d3ac83e.7z";
-      hash = "sha256-3roQlHgi8CUtLKji2YZLNgo8s92eUv3a+AbKo7VFB2U=";
-    };
-  };
-
-  installSpt = pkgs.writeShellScript "install-spt" ''
-    mkdir spt
-    cd spt
-    ${pkgs.p7zip}/bin/7z x -y ${sptAki.release-3_8_0}
-    ${pkgs.p7zip}/bin/7z x -y ${sptAki.update-3_8_1}
-  '';
-in
-
-{
-  links.${cid} = {
-    protocol = "http";
-    ipv4 = config.reflection.interfaces.primary.addrPublic;
-  };
-
-  users.users.${cid} = {
-    isNormalUser = true;
-    group = cid;
-    inherit home;
-    openssh.authorizedKeys.keys = [
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvaZG+5642/jjqVaFGsS3DK0Jtg1wfMY4Yh20tFKtdcZEsLRy16KSJkPH447vP91pGkx8T+GJ1kXEw4dMR9dOKDwS2qCgHwAQbZ2V+/NNJ56bCzTo+geSc/imrWjiHQUhzPRcTyI6pVi3rVAhzEnAVjcC7a4LnLsIFW8Ill0kF8OR4tHoeDNCNN/0XOgZH4dIT9eQCyw4u5zOaw9W81eTL7K1PLmbAWW1+qd//C6CrpxpxNO/kAnOavkscK1uJSfGRcs7Vb5mKKV2J7Be0vxCVY8C8h1qcYcrzudTs9MbQAgFP00bVcvl/byd5CAccQ2wBfuwUdJBC4MV8HZ152biGds8sdCBDoxyvSqScaBFza0iHAXPEygvfx+gEW9KNssfBoiU50SJgWbQUp9gQoppATg8XHqUI03xUeVBRbTyf7UMd2Qk3slWPEnEXlzvjwKGxyLsI0WKdWtqALgKe7mDn+wDAG15CayACi4kgKkZh3VariRC5Ks17LJdLH67vBvs="
-    ];
-  };
-  users.groups.${cid} = {};
-
-  systemd.services.${cid} = {
-    wantedBy = [ "multi-user.target" ];
-
-    path = [
-      pkgs.jq
-      pkgs.wine64Packages.staging
-      pkgs.tmux
-    ];
-
-    preStart = ''
-      cd ${home}
-      test -e drive_c || wine64 wineboot
-
-      cd drive_c
-      test -e spt || ${installSpt}
-
-      cd spt
-      jq < Aki_Data/Server/configs/http.json > .http-new.json \
-        '.ip = "${link.ipv4}" | .port = ${link.portStr} | .backendIp = "${link.ipv4}" | .backendPort = ${link.portStr}'
-      mv .http-new.json Aki_Data/Server/configs/http.json
-    '';
-
-    script = ''
-      cd ${home}/drive_c/spt
-      tmux new -s 0 -d wine64 Aki.Server.exe
-      exec tmux wait-for stop
-    '';
-
-    environment = {
-      WINEPREFIX = "${home}";
-    };
-
-    restartIfChanged = false;
-
-    serviceConfig = {
-      DynamicUser = true;
-      User = cid;
-      Group = cid;
-      ReadWritePaths = [ home ];
-
-      ExecStop = "${pkgs.wine64Packages.staging}/bin/wineserver --kill";
-      Restart = "on-failure";
-
-      CPUQuota = "75%";
-      MemoryMax = "2G";
-      MemorySwapMax = "2G";
-
-      IPAddressDeny = [
-        "10.0.0.0/8"
-        "100.64.0.0/10"
-        "169.254.0.0/16"
-        "172.16.0.0/12"
-        "192.0.0.0/24"
-        "192.0.2.0/24"
-        "192.168.0.0/16"
-        "198.18.0.0/15"
-        "198.51.100.0/24"
-        "203.0.113.0/24"
-        "240.0.0.0/4"
-        "100::/64"
-        "2001:2::/48"
-        "2001:db8::/32"
-        "fc00::/7"
-        "fe80::/10"
-      ];
-      IPAddressAllow = lib.unique config.networking.nameservers;
-    };
-  };
-
-  systemd.services."${cid}-backup" = {
-    startAt = "04:00";
-
-    script = ''
-      cd ${home}/drive_c/spt/user
-      tarball=".profiles-backup-$(date +%s).tar"
-      final="profiles-backup-$(date +%Y-%m-%d-%H:%M:%S).tar.xz"
-      ${pkgs.gnutar}/bin/tar cvf "$tarball" profiles/
-      ${pkgs.xz}/bin/xz -9 "$tarball"
-      mv "''${tarball}.xz" "$final"
-      ${pkgs.rotate-backups}/bin/rotate-backups -S yes -q --daily 30 --weekly 12 -I 'profiles-backup-*.tar.xz' .
-    '';
-
-    unitConfig.ConditionPathExists = "${home}/drive_c/spt/user";
-
-    serviceConfig = {
-      Type = "oneshot";
-      DynamicUser = true;
-      User = cid;
-      Group = cid;
-      ReadWritePaths = [ home ];
-      PrivateNetwork = true;
-    };
-  };
-
-  systemd.services."${cid}-auto-restart" = {
-    startAt = "05:00";
-
-    script = ''
-      echo -n "Service status: "
-      if ! systemctl is-active '${cid}.service'; then
-        echo Service not active.
-        exit
-      fi
-
-      for i in {1..120}; do
-        if test "$(${pkgs.iproute2}/bin/ss -H -tn 'cgroup = /sys/fs/cgroup/system.slice/${cid}.service' | wc -l)" != 0; then
-          echo Service in use.
-          exit
-        fi
-        sleep 1
-      done
-
-      echo Restarting service...
-      systemctl restart --no-block '${cid}.service'
-    '';
-
-    unitConfig.ConditionPathExists = "${home}/drive_c/spt";
-
-    serviceConfig = {
-      Type = "oneshot";
-    };
-  };
-  systemd.services."${cid}-control" = {
-    script = ''
-      if test -e ${home}/drive_c/spt/control/restart; then
-        echo Action: restart
-        trap 'rm -f ${home}/drive_c/spt/control/restart' EXIT
-        systemctl restart ${cid}.service
-      elif test -e ${home}/drive_c/spt/control/shutdown; then
-        echo Action: stop
-        systemctl stop ${cid}.service
-      else
-        echo Action: start
-        systemctl start ${cid}.service
-      fi
-    '';
-
-    unitConfig.ConditionPathExists = "${home}/drive_c/spt/control";
-
-    serviceConfig = {
-      Type = "oneshot";
-    };
-  };
-
-  systemd.paths."${cid}-control" = {
-    wantedBy = [ "paths.target" ];
-    pathConfig.PathChanged = "${home}/drive_c/spt/control";
-  };
-
-  services.openssh.extraConfig = ''
-    Match User ${cid}
-      ChrootDirectory ${root}
-      ForceCommand internal-sftp -d /pfx/drive_c
-      AllowTcpForwarding no
-      X11Forwarding no
-      PasswordAuthentication no
-  '';
-
-  networking.firewall.allowedTCPPorts = [ link.port ];
-}

cluster/services/consul/default.nix

@@ -22,6 +22,11 @@ in
       ];
       ready = ./ready.nix;
     };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" ];
+      settings = ./test.nix;
+    };
   };
 
   dns.records."consul-remote.internal".consulService = "consul-remote";

cluster/services/consul/test.nix

@@ -0,0 +1,19 @@
+{
+  testScript = ''
+    import json
+
+    start_all()
+
+    with subtest("should form cluster"):
+      for machine in machines:
+        machine.succeed("systemctl start consul-ready.service")
+      for machine in machines:
+        consulConfig = json.loads(machine.succeed("cat /etc/consul.json"))
+        addr = consulConfig["addresses"]["http"]
+        port = consulConfig["ports"]["http"]
+        setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
+        memberList = machine.succeed(f"{setEnv} consul members --status=alive")
+        for machine2 in machines:
+          assert machine2.name in memberList
+  '';
+}

cluster/services/forge/default.nix

@@ -19,19 +19,21 @@
 
   ways.forge.target = let
     host = builtins.head config.services.forge.nodes.server;
-  in config.hostLinks.${host}.forge.url;
+  in config.lib.forService "forge" config.hostLinks.${host}.forge.url;
 
-  garage = {
+  garage = config.lib.forService "forge" {
     keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
     buckets.forgejo.allow.forgejo = [ "read" "write" ];
   };
 
-  monitoring.blackbox.targets.forge = {
+  monitoring.blackbox.targets.forge = config.lib.forService "forge" {
     address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
     module = "https2xx";
   };
 
-  dns.records."ssh.forge".target = map
-    (node: depot.hours.${node}.interfaces.primary.addrPublic)
-    config.services.forge.nodes.server;
+  dns.records = config.lib.forService "forge" {
+    "ssh.forge".target = map
+      (node: depot.hours.${node}.interfaces.primary.addrPublic)
+      config.services.forge.nodes.server;
+  };
 }

cluster/services/frangiclave/default.nix

@@ -0,0 +1,27 @@
+{ config, ... }:
+
+{
+  services.frangiclave = {
+    nodes = {
+      server = [ "VEGAS" "grail" "prophet" ];
+      cluster = config.services.frangiclave.nodes.server;
+      agent = []; # all nodes, for vault-agent, secret templates, etc.
+    };
+    meshLinks = {
+      server.link.protocol = "http";
+      cluster.link.protocol = "http";
+    };
+    nixos = {
+      server = [
+        ./server.nix
+      ];
+      cluster = [];
+      agent = [];
+    };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" "consul" ];
+      settings = ./test.nix;
+    };
+  };
+}

cluster/services/frangiclave/server.nix

@@ -0,0 +1,34 @@
+{ cluster, config, depot, lib, ... }:
+
+let
+  apiLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server;
+  clusterLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-cluster;
+in
+
+{
+  services.vault = {
+    enable = true;
+    package = depot.packages.openbao;
+    address = apiLink.tuple;
+    extraConfig = /*hcl*/ ''
+      api_addr = "${apiLink.url}"
+      cluster_addr = "${clusterLink.url}"
+    '';
+
+    storageBackend = "raft";
+    storageConfig = /*hcl*/ ''
+      node_id = "x${builtins.hashString "sha256" "frangiclave-node-${config.networking.hostName}"}"
+      ${
+        lib.pipe (cluster.config.services.frangiclave.otherNodes.server config.networking.hostName) [
+          (map (node: cluster.config.hostLinks.${node}.frangiclave-server))
+          (map (link: /*hcl*/ ''
+            retry_join {
+              leader_api_addr = "${link.url}"
+            }
+          ''))
+          (lib.concatStringsSep "\n")
+        ]
+      }
+    '';
+  };
+}

cluster/services/frangiclave/test.nix

@@ -0,0 +1,12 @@
+{ lib, ... }:
+
+{
+  interactive.defaults = { cluster, config, ... }: {
+    config = lib.mkIf config.services.vault.enable {
+      environment.variables.VAULT_ADDR = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server.url;
+      environment.systemPackages = [ config.services.vault.package ];
+    };
+  };
+
+  testScript = "assert False";
+}

cluster/services/hercules-ci-multi-agent/common.nix

@@ -1,4 +1,4 @@
-{ cluster, depot, lib, ... }:
+{ cluster, config, depot, lib, ... }:
 
 let
   inherit (cluster.config.services.hercules-ci-multi-agent) nodes secrets;
@@ -40,8 +40,19 @@ in
     settings = {
       clusterJoinTokenPath = secrets."clusterJoinToken-${org}".path;
       binaryCachesPath = secrets.cacheConfig.path;
+      concurrentTasks = lib.pipe config.reflection.hardware.cpu.cores [
+        (lib.flip builtins.div 2)
+        builtins.floor
+        (lib.max 2)
+      ];
     };
   });
 
+  nix.settings.cores = lib.pipe config.reflection.hardware.cpu.cores [
+    (builtins.mul 0.75)
+    builtins.floor
+    (lib.max 1)
+  ];
+
   users.groups.hercules-ci-agent.members = map (org: "hci-${org}") (lib.attrNames nodes);
 }

cluster/services/idm/client.nix

@@ -27,7 +27,7 @@ in
     mode = "0755";
     text = ''
       #!/bin/sh
-      exec ${pkgs.kanidm}/bin/kanidm_ssh_authorizedkeys "$@"
+      exec ${config.services.kanidm.package}/bin/kanidm_ssh_authorizedkeys "$@"
     '';
   };
 
@@ -57,10 +57,10 @@ in
   environment.systemPackages = let
     idmAlias = pkgs.runCommand "kanidm-idm-alias" {} ''
       mkdir -p $out/bin
-      ln -s ${pkgs.kanidm}/bin/kanidm $out/bin/idm
+      ln -s ${config.services.kanidm.package}/bin/kanidm $out/bin/idm
       mkdir -p $out/share/bash-completion/completions
       cat >$out/share/bash-completion/completions/idm.bash <<EOF
-      source ${pkgs.kanidm}/share/bash-completion/completions/kanidm.bash
+      source ${config.services.kanidm.package}/share/bash-completion/completions/kanidm.bash
       complete -F _kanidm -o bashdefault -o default idm
       EOF
     '';

cluster/services/irc/irc-host.nix

@@ -90,7 +90,7 @@ in {
   security.pam.services.ngircd = {
     text = ''
       # verify IRC users via IDM
-      auth required ${pkgs.kanidm}/lib/pam_kanidm.so 
+      auth required ${config.services.kanidm.package}/lib/pam_kanidm.so 
     '';
   };
   systemd.services.ngircd = {

cluster/services/storage/default.nix

@@ -35,6 +35,8 @@ in
         ./garage.nix
         ./garage-options.nix
         ./garage-layout.nix
+      ] ++ lib.optionals config.simulacrum [
+        ./simulacrum/snakeoil-rpc-secret.nix
       ];
       garageConfig = [
         ./garage-gateway.nix
@@ -48,6 +50,11 @@ in
       garageInternal = [ ./garage-internal.nix ];
       garageExternal = [ ./garage-external.nix ];
     };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" "consul" "locksmith" ];
+      settings = ./test.nix;
+    };
   };
 
   links = {

cluster/services/storage/garage.nix

@@ -63,6 +63,8 @@ in
   };
 
   systemd.services.garage = {
+    requires = [ "consul-ready.service" ];
+    after = [ "consul-ready.service" ];
     unitConfig = {
       RequiresMountsFor = [ cfg.settings.data_dir ];
     };

cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix

@@ -0,0 +1,3 @@
+{
+  environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+}

cluster/services/storage/test.nix

@@ -0,0 +1,106 @@
+{ cluster, lib, ... }:
+
+let
+  inherit (cluster.config.services.storage) nodes;
+
+  firstGarageNode = lib.elemAt nodes.garage 0;
+in
+
+{
+  nodes = lib.genAttrs nodes.garage (node: {
+    services.garage = {
+      layout.initial = lib.genAttrs nodes.garage (_: {
+        capacity = lib.mkOverride 51 1000;
+      });
+    };
+    specialisation.modifiedLayout = {
+      inheritParentConfig = true;
+      configuration = {
+        services.garage = {
+          layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
+          keys.testKey.allow.createBucket = true;
+          buckets = {
+            bucket1 = {
+              allow.testKey = [ "read" "write" ];
+              quotas = {
+                maxObjects = 300;
+                maxSize = 400 * 1024 * 1024;
+              };
+            };
+            bucket2 = {
+              allow.testKey = [ "read" ];
+            };
+          };
+        };
+        system.ascensions.garage-layout.incantations = lib.mkForce (i: [
+          (i.runGarage ''
+            garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
+            garage layout apply --version 2
+          '')
+        ]);
+      };
+    };
+  });
+
+  testScript = ''
+    import json
+    nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
+    garage1 = nodes[0]
+
+    start_all()
+
+    with subtest("should bootstrap new cluster"):
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+    with subtest("should apply new layout with ascension"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
+
+      for node in nodes:
+        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should apply new layout from scratch"):
+      for node in nodes:
+        node.systemctl("stop garage.service")
+        node.succeed("rm -rf /var/lib/garage-metadata")
+
+      for node in nodes:
+        node.systemctl("start garage.service")
+
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should create specified buckets and keys"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
+      garage1.succeed("garage key list | grep testKey")
+      garage1.succeed("garage bucket list | grep bucket1")
+      garage1.succeed("garage bucket list | grep bucket2")
+
+    with subtest("should delete unspecified buckets and keys"):
+      garage1.succeed("garage bucket create unwantedbucket")
+      garage1.succeed("garage key new --name unwantedkey")
+      garage1.succeed("systemctl restart garage-apply.service")
+
+      garage1.fail("garage key list | grep unwantedkey")
+      garage1.fail("garage bucket list | grep unwantedbucket")
+  '';
+}
+

cluster/services/wireguard/default.nix

@@ -10,6 +10,19 @@ let
   };
 
   getExtAddr = host: host.interfaces.primary.addrPublic;
+
+  snakeoilPublicKeys = {
+    checkmate = "TESTtbFybW5YREwtd18a1A4StS4YAIUS5/M1Lv0jHjA=";
+    grail = "TEsTh7bthkaDh9A1CpqDi/F121ao5lRZqIJznLH8mB4=";
+    thunderskin = "tEST6afFmVN18o+EiWNFx+ax3MJwdQIeNfJSGEpffXw=";
+    VEGAS = "tEsT6s7VtM5C20eJBaq6UlQydAha8ATlmrTRe9T5jnM=";
+    prophet = "TEstYyb5IoqSL53HbSQwMhTaR16sxcWcMmXIBPd+1gE=";
+  };
+
+  grease = hourName: realPublicKey: if config.simulacrum then
+    snakeoilPublicKeys.${hourName}
+  else
+    realPublicKey;
 in
 {
   vars = {
@@ -22,7 +35,7 @@ in
       extra = {
         meshIp = "10.1.1.32";
         inherit meshNet;
-        pubKey = "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
+        pubKey = grease "checkmate" "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
         extraRoutes = [];
       };
     };
@@ -31,7 +44,7 @@ in
       extra = {
         meshIp = "10.1.1.6";
         inherit meshNet;
-        pubKey = "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
+        pubKey = grease "grail" "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
         extraRoutes = [];
       };
     };
@@ -40,7 +53,7 @@ in
       extra = {
         meshIp = "10.1.1.4";
         inherit meshNet;
-        pubKey = "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
+        pubKey = grease "thunderskin" "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
         extraRoutes = [];
       };
     };
@@ -49,7 +62,7 @@ in
       extra = {
         meshIp = "10.1.1.5";
         inherit meshNet;
-        pubKey = "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
+        pubKey = grease "VEGAS" "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
         extraRoutes = [ "${hours.VEGAS.interfaces.vstub.addr}/32" "10.10.0.0/16" ];
       };
     };
@@ -58,7 +71,7 @@ in
       extra = {
         meshIp = "10.1.1.9";
         inherit meshNet;
-        pubKey = "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
+        pubKey = grease "prophet" "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
         extraRoutes = [];
       };
     };
@@ -69,8 +82,12 @@ in
       storm = [ "VEGAS" ];
     };
     nixos = {
-      mesh = ./mesh.nix;
-      storm = ./storm.nix;
+      mesh = [
+        ./mesh.nix
+      ] ++ lib.optionals config.simulacrum [
+        ./simulacrum/snakeoil-keys.nix
+      ];
+      storm = [ ./storm.nix ];
     };
     secrets.meshPrivateKey = {
       nodes = config.services.wireguard.nodes.mesh;

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-VEGAS

@@ -0,0 +1 @@
+MNvWpMluuzQvPyGTp7jtyPSyz6n9lIly/WX1gW2NAHg=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-checkmate

@@ -0,0 +1 @@
+YHzP8rBP6qiXs6ZdnvHop9KnCYRADIEejwZzAzvj8m4=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-grail

@@ -0,0 +1 @@
+uD7X5E6N9d0sN+xPr/bWnehSa3bAok741GO7Z4I+Z3I=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-prophet

@@ -0,0 +1 @@
+QHyIJ3HoKGGFN28qOrQP4UyoQMP5bM7Idn2MzayKzEM=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-thunderskin

@@ -0,0 +1 @@
+YLl+hkWaCWx/5PpWs3cQ+bKqYdJef/qZ+FMTsM9ammM=

cluster/services/wireguard/simulacrum/snakeoil-keys.nix

@@ -0,0 +1,6 @@
+{ lib, config, ... }: {
+  config.environment.etc = {
+    "dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
+    "dummy-secrets/wireguard-key-storm".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
+  };
+}

cluster/simulacrum/checks.nix

@@ -0,0 +1,16 @@
+{ config, extendModules, lib, ... }:
+
+{
+  perSystem = { pkgs, system, ... }: {
+    checks = lib.mkIf (system == "x86_64-linux") (lib.mapAttrs' (name: svc: let
+      runSimulacrum = pkgs.callPackage ./. {
+        inherit config extendModules;
+      };
+    in {
+      name = "simulacrum-${name}";
+      value = runSimulacrum {
+        service = name;
+      };
+    }) (lib.filterAttrs (_: svc: svc.simulacrum.enable) config.cluster.config.services));
+  };
+}

cluster/simulacrum/default.nix

@@ -0,0 +1,117 @@
+{ testers, config, extendModules, lib, system }:
+
+{ service }:
+
+let
+  serviceConfig = config.cluster.config.services.${service};
+  serviceList = [ service ] ++ serviceConfig.simulacrum.deps;
+  allAugments = map (svc: config.cluster.config.services.${svc}.simulacrum.augments) serviceList;
+
+  lift = config;
+
+  snakeoil = {
+    ssh = {
+      public = lib.fileContents ../../packages/checks/snakeoil/ssh/snakeoil-key.pub;
+      private = ../../packages/checks/snakeoil/ssh/snakeoil-key;
+    };
+  };
+
+  nodes = lib.attrNames config.gods.fromLight;
+  digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes (lib.range 1 255));
+  depot' = extendModules {
+    modules = [
+      ({ config, ... }: {
+        gods.fromLight = lib.mapAttrs (name: cfg: {
+          interfaces.primary = {
+            link = lib.mkForce "vprimary";
+          };
+          ssh.id.publicKey = lib.mkForce snakeoil.ssh.public;
+        }) lift.gods.fromLight;
+
+        cluster = lib.mkForce (lift.cluster.extendModules {
+          specialArgs.depot = config;
+          modules = [
+            {
+              simulacrum = true;
+              testConfig = {
+                subject = service;
+                activeServices = serviceList;
+              };
+            }
+          ];
+        });
+      })
+    ];
+  };
+  specialArgs = depot'.config.lib.summon system lib.id;
+in
+
+testers.runNixOSTest {
+  name = "simulacrum-${service}";
+
+  imports = [
+    serviceConfig.simulacrum.settings
+  ] ++ allAugments;
+
+  _module.args = {
+    inherit (depot'.config) cluster;
+  };
+
+  node = { inherit specialArgs; };
+  nodes = lib.genAttrs nodes (node: let
+    hour = depot'.config.hours.${node};
+  in {
+    imports = [
+      specialArgs.depot.hours.${node}.nixos
+      ../../packages/checks/modules/nixos/age-dummy-secrets
+      ../../packages/checks/modules/nixos/external-storage.nix
+    ] ++ depot'.config.cluster.config.out.injectNixosConfigForServices serviceList node;
+
+    boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
+    networking = {
+      interfaces = {
+        ${hour.interfaces.primary.link} = {
+          useDHCP = lib.mkForce false;
+          virtual = true;
+          ipv4.addresses = lib.mkForce [
+            {
+              address = hour.interfaces.primary.addr;
+              prefixLength = 32;
+            }
+          ];
+        };
+        eth1.ipv4.routes = lib.pipe nodes [
+          (lib.filter (n: n != node))
+          (map (n: let
+            hour = depot'.config.hours.${n};
+          in {
+            address = hour.interfaces.primary.addrPublic;
+            prefixLength = 32;
+            via = "192.168.1.${toString digits.${n}}";
+          }))
+        ];
+      };
+
+      firewall.extraCommands = lib.mkAfter (lib.optionalString (hour.interfaces.primary.isNat) ''
+        # self-nat
+        iptables -t nat -A PREROUTING -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
+        iptables -t nat -A POSTROUTING -s ${hour.interfaces.primary.addr} -j SNAT --to-source ${hour.interfaces.primary.addrPublic}
+      '');
+    };
+
+    systemd.services = {
+      hyprspace.enable = false;
+    };
+
+    environment.etc = {
+      "ssh/ssh_host_ed25519_key" = {
+        source = snakeoil.ssh.private;
+        mode = "0400";
+      };
+    };
+    virtualisation = {
+      cores = 2;
+      memorySize = 4096;
+    };
+  });
+}

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1718371084,
-        "narHash": "sha256-abpBi61mg0g+lFFU0zY4C6oP6fBwPzbHPKBGw676xsA=",
+        "lastModified": 1722339003,
+        "narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "3a56735779db467538fb2e577eda28a9daacaca6",
+        "rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
         "type": "github"
       },
       "original": {
@@ -40,11 +40,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717279440,
-        "narHash": "sha256-kH04ReTjxOpQumgWnqy40vvQLSnLGxWP6RF3nq5Esrk=",
+        "lastModified": 1722472866,
+        "narHash": "sha256-GJIz4M5HDB948Ex/8cPvbkrNzl/eKUE7/c21JBu4lb8=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "717cc95983cdc357bc347d70be20ced21f935843",
+        "rev": "e127acbf9a71ebc0c26bc8e28346822e0a6e16ba",
         "type": "github"
       },
       "original": {
@@ -113,19 +113,16 @@
     },
     "devshell": {
       "inputs": {
-        "flake-utils": [
-          "repin-flake-utils"
-        ],
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1717408969,
-        "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=",
+        "lastModified": 1722113426,
+        "narHash": "sha256-Yo/3loq572A8Su6aY5GP56knpuKYRvM2a1meP9oJZCw=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "1ebbe68d57457c8cae98145410b164b5477761f4",
+        "rev": "67cce7359e4cd3c45296fb4aaf6a19e2a9c757ae",
         "type": "github"
       },
       "original": {
@@ -197,11 +194,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1719994518,
-        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
         "type": "github"
       },
       "original": {
@@ -271,11 +268,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1719398431,
-        "narHash": "sha256-p6cC+0/c6GCBPtBkSnaOrhwLWl+uyoWYl/dReg+jcsk=",
+        "lastModified": 1720223941,
+        "narHash": "sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4=",
         "owner": "hercules-ci",
         "repo": "hercules-ci-agent",
-        "rev": "c37b2ada2dd001bc4be6771bcdea680b0b93fb94",
+        "rev": "2e10fb21fc2e07edf40763b73443e5934bd40947",
         "type": "github"
       },
       "original": {
@@ -336,11 +333,11 @@
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1721065874,
-        "narHash": "sha256-WqIRfqqQwcSrJNPVRxbvCG6uZvKNuPruBL85VcnESRA=",
+        "lastModified": 1721084841,
+        "narHash": "sha256-zWajCfHFqPa3Z72DHcxBUq4bmcCu1lpEKUbZZewpYOE=",
         "owner": "hyprspace",
         "repo": "hyprspace",
-        "rev": "76a3f73a42c2f9bbeb4c56afa4b30a98a283b79f",
+        "rev": "b54fd70812b98994630cfa6aac17ad7c2be9b468",
         "type": "github"
       },
       "original": {
@@ -396,19 +393,17 @@
     },
     "nar-serve": {
       "inputs": {
-        "flake-utils": [
-          "repin-flake-utils"
-        ],
         "nixpkgs": [
           "nixpkgs"
-        ]
+        ],
+        "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1716758395,
-        "narHash": "sha256-yM/ICgmMxUAk/feKojy/Jul8jh4OaVBhQoIChA6Vvq8=",
+        "lastModified": 1722409392,
+        "narHash": "sha256-8QuMS00EutmqzAIPxyJEPxM8EHiWlSKs6E2Htoh3Kes=",
         "owner": "numtide",
         "repo": "nar-serve",
-        "rev": "a1458804bb1ab9f1a44101e56a010ca95b8e8309",
+        "rev": "9d0eff868d328fe67c60c26c8ba50e0b9d8de867",
         "type": "github"
       },
       "original": {
@@ -461,11 +456,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1719254875,
-        "narHash": "sha256-ECni+IkwXjusHsm9Sexdtq8weAq/yUyt1TWIemXt3Ko=",
+        "lastModified": 1719848872,
+        "narHash": "sha256-H3+EC5cYuq+gQW8y0lSrrDZfH71LB4DAf+TDFyvwCNA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2893f56de08021cffd9b6b6dfc70fd9ccd51eb60",
+        "rev": "00d80d13810dbfea8ab4ed1009b09100cca86ba8",
         "type": "github"
       },
       "original": {
@@ -509,11 +504,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1720107418,
-        "narHash": "sha256-KPxjth7cCwNeTV9jJ90lPu8b/wMi3hcAnUbZ5NzogjA=",
+        "lastModified": 1722539632,
+        "narHash": "sha256-g4L+I8rDl7RQy5x8XcEMqNO49LFhrHTzVBqXtG2+FGo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d586b644539e41fafcee09f9b40a6552252dbdb4",
+        "rev": "f2d6c7123138044e0c68902268bd8f37dd7e2fa7",
         "type": "github"
       },
       "original": {
@@ -592,7 +587,7 @@
         "nix-super": "nix-super",
         "nixpkgs": "nixpkgs_4",
         "repin-flake-utils": "repin-flake-utils",
-        "systems": "systems_2"
+        "systems": "systems_3"
       }
     },
     "systems": {
@@ -611,6 +606,21 @@
       }
     },
     "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
       "locked": {
         "lastModified": 1683988522,
         "narHash": "sha256-cAVqq+Mjx4EiWovT7rwbwncX5QBnhNsqwsgyg+eeYrg=",

flake.nix

@@ -62,7 +62,6 @@
       url = "github:numtide/nar-serve";
       inputs = {
         nixpkgs.follows = "nixpkgs";
-        flake-utils.follows = "repin-flake-utils";
       };
     };
 
@@ -70,7 +69,6 @@
       url = "github:numtide/devshell";
       inputs = {
         nixpkgs.follows = "nixpkgs";
-        flake-utils.follows = "repin-flake-utils";
       };
     };
     

hosts/VEGAS/default.nix

@@ -16,6 +16,11 @@ tools: rec {
     };
   };
 
+  hardware = {
+    cpu.cores = 8;
+    memory.gb = 64;
+  };
+
   hyprspace = {
     enable = true;
     id = "QmYs4xNBby2fTs8RnzfXEk161KD4mftBfCiR8yXtgGPj4J";

hosts/checkmate/default.nix

@@ -17,6 +17,11 @@ tools: rec {
     };
   };
 
+  hardware = {
+    cpu.cores = 1;
+    memory.gb = 1;
+  };
+
   hyprspace = {
     enable = true;
     id = "12D3KooWL84sAtq1QTYwb7gVbhSNX5ZUfVt4kgYKz8pdif1zpGUh";

hosts/grail/default.nix

@@ -16,6 +16,11 @@ tools: rec {
     };
   };
 
+  hardware = {
+    cpu.cores = 4;
+    memory.gb = 8;
+  };
+
   hyprspace = {
     enable = true;
     id = "12D3KooWN31twBvdEcxz2jTv4tBfPe3mkNueBwDJFCN4xn7ZwFbi";

hosts/options/default.nix

@@ -9,6 +9,7 @@ let
       ./hour/interfaces.nix
       ./hour/nixos.nix
       ./hour/ssh.nix
+      ./hour/hardware.nix
     ];
   };
 

hosts/options/hour/hardware.nix

@@ -0,0 +1,18 @@
+{ lib, ... }:
+with lib;
+
+{
+  options.hardware = {
+    cpu = {
+      cores = mkOption {
+        type = types.ints.unsigned;
+      };
+    };
+    memory = {
+      gb = mkOption {
+        type = types.ints.unsigned;
+      };
+    };
+  };
+}
+

hosts/prophet/default.nix

@@ -17,6 +17,11 @@ tools: rec {
     };
   };
 
+  hardware = {
+    cpu.cores = 4;
+    memory.gb = 24;
+  };
+
   hyprspace = {
     enable = true;
     id = "QmbrAHuh4RYcyN9fWePCZMVmQjbaNXtyvrDCWz4VrchbXh";

hosts/thunderskin/default.nix

@@ -17,6 +17,11 @@ tools: rec {
     };
   };
 
+  hardware = {
+    cpu.cores = 1;
+    memory.gb = 1;
+  };
+
   hyprspace = {
     enable = true;
     id = "12D3KooWB9AUPorFoACkWbphyargRBV9osJsYuQDumtQ85j7Aqmg";

packages/catalog/checks.nix

@@ -1,21 +1,32 @@
 { lib, ... }:
 
 {
-  perSystem = { config, ... }: {
-    catalog.depot = {
-      checks = lib.mapAttrs (name: check: {
-        description = "NixOS Test: ${name}";
-        actions = {
-          build = {
-            description = "Build this check.";
-            command = "nix build -L --no-link '${builtins.unsafeDiscardStringContext check.drvPath}^*'";
-          };
-          runInteractive = {
-            description = "Run interactive driver.";
-            command = lib.getExe check.driverInteractive;
-          };
+  perSystem = { config, pkgs, ... }: {
+    catalog = lib.mkMerge (lib.mapAttrsToList (name': check: let
+      simulacrum = lib.hasPrefix "simulacrum-" name';
+      name = lib.removePrefix "simulacrum-" name';
+      baseAttrPath = if simulacrum then
+        [ "cluster" "simulacrum" ]
+      else
+        [ "depot" "checks" ];
+    in lib.setAttrByPath (baseAttrPath ++ [ name ]) {
+      description = if simulacrum then
+        "Simulacrum Test: ${name}"
+      else
+        "NixOS Test: ${name}";
+      actions = {
+        build = {
+          description = "Build this check.";
+          command = "nix build -L --no-link '${builtins.unsafeDiscardStringContext check.drvPath}^*'";
         };
-      }) config.checks;
-    };
+        runInteractive = {
+          description = "Run interactive driver.";
+          command = if simulacrum then
+            "${pkgs.bubblewrap}/bin/bwrap --unshare-all --bind / / --dev-bind /dev /dev ${lib.getExe check.driverInteractive}"
+          else
+            lib.getExe check.driverInteractive;
+        };
+      };
+    }) config.checks);
   };
 }

packages/catalog/packages.nix

@@ -8,7 +8,7 @@ in
   perSystem = { config, ... }: {
     catalog.depot = {
       packages = lib.mapAttrs (name: package: {
-        description = package.meta.description or "Package: ${name}";
+        description = "Package: ${name}";
         actions = lib.mkMerge [
           {
             build = {

packages/checks/default.nix

@@ -7,6 +7,7 @@ let
 in
 
 {
+  debug = lib.warn "debug mode is enabled" true;
   perSystem = { filters, pkgs, self', system, ... }: {
     checks = lib.mkIf (system == "x86_64-linux") {
       ascensions = pkgs.callPackage ./ascensions.nix {

packages/checks/modules/nixos/external-storage.nix

@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+
+{
+  systemd.tmpfiles.settings."00-testing-external-storage-underlays" = lib.mapAttrs' (name: cfg: {
+    name = cfg.mountpoint;
+    value.d = {
+      user = toString cfg.uid;
+      group = toString cfg.gid;
+      mode = "0700";
+    };
+  }) config.services.external-storage.underlays;
+}

packages/checks/snakeoil/ssh/snakeoil-key

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgwAAAJAS78fWEu/H
+1gAAAAtzc2gtZWQyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgw
+AAAEAUtGOZZIZdzGP6g85JuXBjDtciNQ9bLHNxSN5Gbwvb2Q7HTdf4u1bRo3x6N03ggmAM
++tNmBXB7tZteGEG+pXCDAAAACW1heEBUSVRBTgECAwQ=
+-----END OPENSSH PRIVATE KEY-----

packages/checks/snakeoil/ssh/snakeoil-key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA7HTdf4u1bRo3x6N03ggmAM+tNmBXB7tZteGEG+pXCD

packages/shadows.nix

@@ -9,11 +9,7 @@
 
     options.shadows = lib.mkOption {
       type = with lib.types; lazyAttrsOf package;
-      default = {
-        inherit (self'.packages)
-          kanidm
-        ;
-      };
+      default = { };
     };
   };
 }

packages/tools/npins/default.nix

@@ -21,7 +21,7 @@ in rustPlatform.buildRustPackage rec {
   inherit (src) version;
   src = passthru.mkSource sources.npins;
 
-  cargoSha256 = "0rwnzkmx91cwcz9yw0rbbqv73ba6ggim9f4qgz5pgy6h696ld2k8";
+  cargoHash = "sha256-aIpGTTLQ+HfLf5i4VON7Rq1xNl4rA+7TZ5yF1Ov8lmc=";
 
   buildInputs = lib.optional stdenv.isDarwin (with darwin.apple_sdk.frameworks; [ Security ]);
   nativeBuildInputs = [ makeWrapper ];

patches/base/s3ql/remove-ssl-monkeypatch.patch

@@ -0,0 +1,12 @@
+diff --git a/tests/t0_http.py b/tests/t0_http.py
+index 66ed564f..36bebab1 100755
+--- a/tests/t0_http.py
++++ b/tests/t0_http.py
+@@ -289,7 +289,6 @@ def do_GET(self):
+ 
+     # We don't *actually* want to establish SSL, that'd be
+     # to complex for our mock server
+-    monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
+     conn = HTTPConnection(
+         test_host,
+         test_port,