cluster/services/frangiclave: funny

cluster/services/frangiclave: test in simulacrum WIP
cluster/services/frangiclave: retry_join
2024-08-12 03:04:14 +02:00 · 2024-08-12 03:04:14 +02:00 · 2024-08-12 03:04:14 +02:00 · 2024-08-12 03:04:14 +02:00 · 2024-08-12 03:04:14 +02:00 · 2024-08-12 03:04:03 +02:00
89 changed files with 1327 additions and 1221 deletions
--- a/cluster/catalog/secrets.nix
+++ b/cluster/catalog/secrets.nix
@ -46,6 +46,7 @@ in
              };
            }) // (if secretConfig.shared then let
              secretFile = "${svcName}-${secretName}.age";
+              snakeoilFile = "${svcName}-${secretName}-snakeoil.txt";
            in {
              editSecret = {
                description = "Edit this secret";
@ -54,15 +55,31 @@ in
                  agenix -e '${secretFile}'
                '';
              };
-            } else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
-              secretFile = "${svcName}-${secretName}-${node}.age";
-            in {
-              description = "Edit this secret for '${node}'";
-              command = ''
-                ${setupCommands secretFile [ node ]}
-                agenix -e '${secretFile}'
-              '';
-            })));
+              editSnakeoil = {
+                description = "Edit this secret's snakeoil";
+                command = ''
+                  $EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
+                '';
+              };
+            } else lib.mkMerge [
+              (lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
+                secretFile = "${svcName}-${secretName}-${node}.age";
+              in {
+                description = "Edit this secret for '${node}'";
+                command = ''
+                  ${setupCommands secretFile [ node ]}
+                  agenix -e '${secretFile}'
+                '';
+              })))
+              (lib.mapAttrs' (name: lib.nameValuePair "editSnakeoilInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
+                snakeoilFile = "${svcName}-${secretName}-${node}-snakeoil.txt";
+              in {
+                description = "Edit this secret's snakeoil for '${node}'";
+                command = ''
+                  $EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
+                '';
+              })))
+            ]);
          };
        }) svcConfig.secrets))
        lib.concatLists
--- a/cluster/lib/mesh.nix
+++ b/cluster/lib/mesh.nix
@ -3,14 +3,12 @@
 {
  hostLinks = lib.pipe config.services [
    (lib.filterAttrs (_: svc: svc.meshLinks != {}))
-    (lib.mapAttrsToList (svcName: svc:
-      lib.mapAttrsToList (groupName: links:
-        lib.genAttrs svc.nodes.${groupName} (hostName: lib.mapAttrs (_: cfg: { ... }: {
-          imports = [ cfg.link ];
-          ipv4 = config.vars.mesh.${hostName}.meshIp;
-        }) links)
-      ) svc.meshLinks
-    ))
+    (lib.mapAttrsToList (svcName: svc: lib.mapAttrsToList (name: cfg: lib.genAttrs svc.nodes.${name} (hostName: {
+      ${cfg.name} = { ... }: {
+        imports = [ cfg.link ];
+        ipv4 = config.vars.mesh.${hostName}.meshIp;
+      };
+    })) svc.meshLinks))
    (map lib.mkMerge)
    lib.mkMerge
  ];
--- a/cluster/lib/service-module.nix
+++ b/cluster/lib/service-module.nix
@ -38,8 +38,12 @@ in
    };
    meshLinks = mkOption {
      description = "Create host links on the mesh network.";
-      type = types.attrsOf (types.attrsOf (types.submodule {
+      type = types.attrsOf (types.submodule ({ name, ... }: {
        options = {
+          name = mkOption {
+            type = types.str;
+            default = "${serviceName}-${name}";
+          };
          link = mkOption {
            type = types.deferredModule;
            default = {};
--- a/cluster/services/acme-client/client.nix
+++ b/cluster/services/acme-client/client.nix
@ -74,7 +74,7 @@ in
      serviceConfig = {
        Restart = "on-failure";
        RestartMaxDelaySec = 30;
-        RestartSteps = 5;
+        RestartStesp = 5;
        RestartMode = "direct";
      };
    };
--- a/cluster/services/attic/default.nix
+++ b/cluster/services/attic/default.nix
@ -16,7 +16,10 @@
        ./nar-serve.nix
      ];
    };
-    meshLinks.server.attic.link.protocol = "http";
+    meshLinks.server = {
+      name = "attic";
+      link.protocol = "http";
+    };
    secrets = let
      inherit (config.services.attic) nodes;
    in {
@ -49,12 +52,10 @@
    cache.target = serverAddrs;
  };

-  ways = config.lib.forService "attic" {
-    cache-api = {
-      consulService = "atticd";
-      extras.extraConfig = ''
-        client_max_body_size 4G;
-      '';
-    };
+  ways.cache-api = {
+    consulService = "atticd";
+    extras.extraConfig = ''
+      client_max_body_size 4G;
+    '';
  };
 }
--- a/cluster/services/attic/server.nix
+++ b/cluster/services/attic/server.nix
@ -9,13 +9,17 @@ let
 in

 {
+  imports = [
+    depot.inputs.attic.nixosModules.atticd
+  ];
+
  services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];

  services.atticd = {
    enable = true;
    package = depot.inputs.attic.packages.attic-server;

-    environmentFile = secrets.serverToken.path;
+    credentialsFile = secrets.serverToken.path;
    mode = if isMonolith then "monolithic" else "api-server";

    settings = {
@ -65,7 +69,6 @@ in
    serviceConfig = {
      DynamicUser = lib.mkForce false;
      RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
-      SystemCallFilter = lib.mkAfter [ "@resources" ];
    };
    environment = {
      AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
@ -77,7 +80,6 @@ in
    mode = if isMonolith then "manual" else "direct";
    definition = {
      name = "atticd";
-      id = "atticd-${config.services.atticd.mode}";
      address = link.ipv4;
      inherit (link) port;
      checks = [
--- a/cluster/services/consul/agent.nix
+++ b/cluster/services/consul/agent.nix
@ -10,6 +10,13 @@ let
 in

 {
+  system.extraIncantations = {
+    runConsul = i: script: i.execShellWith [ config.services.consul.package ] ''
+      export CONSUL_HTTP_ADDR='${config.links.consulAgent.tuple}'
+      ${script}
+    '';
+  };
+
  links.consulAgent.protocol = "http";

  services.consul = {
--- a/cluster/services/consul/test.nix
+++ b/cluster/services/consul/test.nix
@ -1,24 +1,19 @@
-{ lib, ... }:
-
 {
-  defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
-
  testScript = ''
    import json

    start_all()

    with subtest("should form cluster"):
-      nodes = [ n for n in machines if n != nowhere ]
-      for machine in nodes:
+      for machine in machines:
        machine.succeed("systemctl start consul-ready.service")
-      for machine in nodes:
+      for machine in machines:
        consulConfig = json.loads(machine.succeed("cat /etc/consul.json"))
        addr = consulConfig["addresses"]["http"]
        port = consulConfig["ports"]["http"]
        setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
        memberList = machine.succeed(f"{setEnv} consul members --status=alive")
-        for machine2 in nodes:
+        for machine2 in machines:
          assert machine2.name in memberList
  '';
 }
--- a/cluster/services/dns/acme-dns-db-credentials.age
+++ b/cluster/services/dns/acme-dns-db-credentials.age
@ -0,0 +1,16 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A YndVtONpmfFXYB1ASnPHsfczl1UbgZ2vccIrX2pEgx0
+VzH2UD583L6wBLMCo6faIGyHR4+zXXOUTgQduEiFOxI
+-> ssh-ed25519 5/zT0w +67r5S6PSFEgnrTu3eZpOd3eemZUdDOE+kjUw6GDgUM
+jPzlW7hePFgsABUjryePu5yergQ2Qjczmmoxuo6CK+U
+-> ssh-ed25519 TCgorQ DGJPjJYpeibxM+8OwofUCdttIT2OdNbvQ66wpWQM8XU
+JCNQ3bT21j2ZsxbzA6FieKIui6lsvk1p0nvNOT7YtFo
+-> ssh-ed25519 d3WGuA hIl5yluwf1f0DP5ZW1MalGPCj4XFYOu2sofwJSQZ6RE
+BSHoe4cdRJlPrkc+taUIaIIUknexlGttzz2d9I3jtmk
+-> ssh-ed25519 YIaSKQ EbqXS/XFQHSXCbzDJmg4gGUxP9TX3+vOxWtNQDJ8ih4
+hNaWzoFG2iVef4Gm30LilGXYNsVkhmVt9dOvBo02mbM
+-> V]i@xRtJ-grease
+NEPxMUZa76GclWOasWptt6QS7frMclp9o+kD4KCLJB7ucFOYK7xxWfAEMkjtadfP
+m0bbgbw7Jcs9/lA8VNAG2D5jTBayGgpkBQZ4
+--- ViqZD8mJEKIMCZ5Q+wRQWR2FX/LMEfUwoumUtHlYabQ
+KAÉû¹ÝgZü<šë*DfV6·=äG»+eœ`ºpª±ï÷6°<1E>º[Û‘Û û¸¢ºÐý-H1<1B>»Ã›Íí[fV.¾¢HÁ"OhÐñŒ½j•ùö8ïßß$‰;Û‘&5<>äxw§/mŒë<C592>Ö‘ß^7î‘f5ÔµyÏŽÓûC‚´6”¹U•æýi-R=/_R<5F><52>„·==æà½1˜'Ò qÞ·ŒvÜcwø	
--- a/cluster/services/dns/coredns.nix
+++ b/cluster/services/dns/coredns.nix
@ -35,13 +35,10 @@ in
    ];
    before = [ "acme-securedns.${domain}.service" ];
    wants = [ "acme-finished-securedns.${domain}.target" ];
-    serviceConfig = {
-      LoadCredential = [
-        "dot-cert.pem:${dot.directory}/fullchain.pem"
-        "dot-key.pem:${dot.directory}/key.pem"
-      ];
-      ExecReload = lib.mkForce [];
-    };
+    serviceConfig.LoadCredential = [
+      "dot-cert.pem:${dot.directory}/fullchain.pem"
+      "dot-key.pem:${dot.directory}/key.pem"
+    ];
  };

  security.acme.certs."securedns.${domain}" = {
--- a/cluster/services/dns/default.nix
+++ b/cluster/services/dns/default.nix
@ -56,11 +56,6 @@ in
      coredns = ./coredns.nix;
      client = ./client.nix;
    };
-    simulacrum = {
-      enable = true;
-      deps = [ "consul" "acme-client" "patroni" ];
-      settings = ./test.nix;
-    };
  };

  patroni = {
--- a/cluster/services/dns/test.nix
+++ b/cluster/services/dns/test.nix
@ -1,35 +0,0 @@
-{ cluster, ... }:
-
-let
-  inherit (cluster._module.specialArgs.depot.lib.meta) domain;
-in
-{
-  nodes.nowhere = { pkgs, ... }: {
-    passthru = cluster;
-    environment.systemPackages = [
-      pkgs.knot-dns
-      pkgs.openssl
-    ];
-  };
-
-  testScript = ''
-    import json
-    nodeNames = json.loads('${builtins.toJSON cluster.config.services.dns.nodes.authoritative}')
-    dotNames = json.loads('${builtins.toJSON cluster.config.services.dns.nodes.coredns}')
-    nodes = [ n for n in machines if n.name in nodeNames ]
-    dotServers = [ n for n in machines if n.name in dotNames ]
-
-    start_all()
-
-    with subtest("should allow external name resolution for own domain"):
-      for node in nodes:
-        node.wait_for_unit("coredns.service")
-      nowhere.wait_until_succeeds("[[ $(kdig +short securedns.${domain} | wc -l) -ne 0 ]]", timeout=60)
-      nowhere.fail("[[ $(kdig +short example.com | wc -l) -ne 0 ]]")
-
-    with subtest("should have valid certificate on DoT endpoint"):
-      for node in dotServers:
-        node.wait_for_unit("acme-finished-securedns.${domain}.target")
-      nowhere.wait_until_succeeds("openssl </dev/null s_client -connect securedns.${domain}:853 -verify_return_error -strict -verify_hostname securedns.${domain}", timeout=60)
-  '';
-}
--- a/cluster/services/flake-reegistry/default.nix
+++ b/cluster/services/flake-reegistry/default.nix
@ -1,36 +0,0 @@
-{ lib, ... }:
-
-{
-  ways.registry.static = { depot, pkgs, ... }: pkgs.writeTextDir "flake-registry.json" (let
-    flakes = {
-      depot = {
-        type = "tarball";
-        url = "https://forge.${depot.lib.meta.domain}/${depot.lib.meta.domain}/depot/archive/master.tar.gz";
-      };
-      depot-nixpkgs = {
-        type = "github";
-        owner = "NixOS";
-        repo = "nixpkgs";
-        inherit (depot.inputs.nixpkgs.sourceInfo) rev narHash lastModified;
-      };
-      blank = {
-        type = "github";
-        owner = "divnix";
-        repo = "blank";
-        inherit (depot.inputs.blank.sourceInfo) rev narHash lastModified;
-      };
-    } // import ./extra-flakes.nix;
-  in builtins.toJSON {
-    version = 2;
-    flakes = lib.pipe flakes [
-      (lib.attrsToList)
-      (map (f: {
-        from = {
-          type = "indirect";
-          id = f.name;
-        };
-        to = f.value;
-      }))
-    ];
-  });
-}
--- a/cluster/services/flake-reegistry/extra-flakes.nix
+++ b/cluster/services/flake-reegistry/extra-flakes.nix
@ -1,18 +0,0 @@
-let
-  github = owner: repo: {
-    type = "github";
-    inherit owner repo;
-  };
-in {
-  # own 
-  hyprspace = github "hyprspace" "hyprspace";
-  ai = github "nixified-ai" "flake";
-  nix-super = github "privatevoid-net" "nix-super";
-  nixpak = github "nixpak" "nixpak";
-
-  # other
-  nix = github "NixOS" "nix";
-  flake-parts = github "hercules-ci" "flake-parts";
-  home-manager = github "nix-community" "home-manager";
-  dream2nix = github "nix-community" "dream2nix";
-}
--- a/cluster/services/forge/default.nix
+++ b/cluster/services/forge/default.nix
@ -4,7 +4,10 @@
  services.forge = {
    nodes.server = [ "VEGAS" ];
    nixos.server = ./server.nix;
-    meshLinks.server.forge.link.protocol = "http";
+    meshLinks.server = {
+      name = "forge";
+      link.protocol = "http";
+    };
    secrets = with config.services.forge.nodes; {
      oidcSecret = {
        nodes = server;
@ -13,10 +16,10 @@
    };
  };

-  ways = let
+  ways.forge = let
    host = builtins.head config.services.forge.nodes.server;
  in config.lib.forService "forge" {
-    forge.target = config.hostLinks.${host}.forge.url;
+    target = config.hostLinks.${host}.forge.url;
  };

  patroni = config.lib.forService "forge" {
--- a/cluster/services/frangiclave/default.nix
+++ b/cluster/services/frangiclave/default.nix
@ -0,0 +1,27 @@
+{ config, ... }:
+
+{
+  services.frangiclave = {
+    nodes = {
+      server = [ "VEGAS" "grail" "prophet" ];
+      cluster = config.services.frangiclave.nodes.server;
+      agent = []; # all nodes, for vault-agent, secret templates, etc.
+    };
+    meshLinks = {
+      server.link.protocol = "http";
+      cluster.link.protocol = "http";
+    };
+    nixos = {
+      server = [
+        ./server.nix
+      ];
+      cluster = [];
+      agent = [];
+    };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" "consul" ];
+      settings = ./test.nix;
+    };
+  };
+}
--- a/cluster/services/frangiclave/server.nix
+++ b/cluster/services/frangiclave/server.nix
@ -0,0 +1,34 @@
+{ cluster, config, depot, lib, ... }:
+
+let
+  apiLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server;
+  clusterLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-cluster;
+in
+
+{
+  services.vault = {
+    enable = true;
+    package = depot.packages.openbao;
+    address = apiLink.tuple;
+    extraConfig = /*hcl*/ ''
+      api_addr = "${apiLink.url}"
+      cluster_addr = "${clusterLink.url}"
+    '';
+
+    storageBackend = "raft";
+    storageConfig = /*hcl*/ ''
+      node_id = "x${builtins.hashString "sha256" "frangiclave-node-${config.networking.hostName}"}"
+      ${
+        lib.pipe (cluster.config.services.frangiclave.otherNodes.server config.networking.hostName) [
+          (map (node: cluster.config.hostLinks.${node}.frangiclave-server))
+          (map (link: /*hcl*/ ''
+            retry_join {
+              leader_api_addr = "${link.url}"
+            }
+          ''))
+          (lib.concatStringsSep "\n")
+        ]
+      }
+    '';
+  };
+}
--- a/cluster/services/frangiclave/test.nix
+++ b/cluster/services/frangiclave/test.nix
@ -0,0 +1,12 @@
+{ lib, ... }:
+
+{
+  interactive.defaults = { cluster, config, ... }: {
+    config = lib.mkIf config.services.vault.enable {
+      environment.variables.VAULT_ADDR = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server.url;
+      environment.systemPackages = [ config.services.vault.package ];
+    };
+  };
+
+  testScript = "assert False";
+}
--- a/cluster/services/gitlab/default.nix
+++ b/cluster/services/gitlab/default.nix
@ -0,0 +1,10 @@
+{ depot, ... }:
+
+{
+  services.gitlab = {
+    nodes.host = [ "VEGAS" ];
+    nixos.host = ./host.nix;
+  };
+
+  dns.records.git.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ];
+}
--- a/cluster/services/gitlab/host.nix
+++ b/cluster/services/gitlab/host.nix
@ -0,0 +1,94 @@
+{ cluster, config, lib, depot, ... }:
+
+let
+  inherit (depot.lib.meta) domain adminEmail;
+
+  patroni = cluster.config.links.patroni-pg-access;
+
+  mkSecret = name: {
+    owner = "gitlab";
+    group = "gitlab";
+    mode = "0400";
+    file = ../../../secrets/${name}.age;
+  };
+
+  secrets = lib.mapAttrs (_: v: v.path) config.age.secrets;
+
+  cfg = config.services.gitlab;
+in
+
+{
+  age.secrets = lib.flip lib.genAttrs mkSecret [
+    "gitlab-db-credentials"
+    "gitlab-initial-root-password"
+    "gitlab-openid-secret"
+    "gitlab-secret-db"
+    "gitlab-secret-jws"
+    "gitlab-secret-otp"
+    "gitlab-secret-secret"
+  ];
+
+  services.gitlab = {
+    enable = true;
+    https = true;
+    host = "git.${domain}";
+    port = 443;
+
+    databaseCreateLocally = false;
+    databaseHost = patroni.ipv4;
+    extraDatabaseConfig = { inherit (patroni) port; };
+    databaseUsername = "gitlab";
+    databasePasswordFile = secrets.gitlab-db-credentials;
+
+    initialRootEmail = adminEmail;
+
+    statePath = "/srv/storage/private/gitlab/state";
+
+    smtp = {
+      enable = true;
+      inherit domain;
+    };
+
+    initialRootPasswordFile = secrets.gitlab-initial-root-password;
+
+    secrets = with secrets; {
+      dbFile = gitlab-secret-db;
+      jwsFile = gitlab-secret-jws;
+      otpFile = gitlab-secret-otp;
+      secretFile = gitlab-secret-secret;
+    };
+
+    extraConfig = {
+      omniauth = {
+        enabled = true;
+        auto_sign_in_with_provider = "openid_connect";
+        allow_single_sign_on = ["openid_connect"];
+        block_auto_created_users = false;
+        providers = [
+
+          {
+            name = "openid_connect";
+            label = "Private Void Account";
+            args = {
+              name = "openid_connect";
+              scope = ["openid" "profile"];
+              response_type = "code";
+              issuer = "https://login.${domain}/auth/realms/master";
+              discovery = true;
+              client_auth_method = "query";
+              uid_field = "preferred_username";
+              client_options = {
+                identifier = "net.privatevoid.git2";
+                secret = { _secret = secrets.gitlab-openid-secret; };
+                redirect_uri = "https://${cfg.host}/users/auth/openid_connect/callback";
+              };
+            };
+          }
+          
+        ];
+      };
+    };
+  };
+
+  services.nginx.virtualHosts."${cfg.host}" = depot.lib.nginx.vhosts.proxy "http://unix:/run/gitlab/gitlab-workhorse.socket";
+}
--- a/cluster/services/idm/client.nix
+++ b/cluster/services/idm/client.nix
@ -1,4 +1,4 @@
-{ cluster, config, lib, pkgs, utils, ... }:
+{ cluster, config, pkgs, utils, ... }:

 let
  frontendLink = cluster.config.links.idm;
@ -39,8 +39,9 @@ in
  security = {
    pam.services.sudo = { config, ... }: {
      rules.auth.rssh = {
-        enable = lib.mkForce true;
        order = config.rules.auth.unix.order - 10;
+        control = "sufficient";
+        modulePath = "${pkgs.pam_rssh}/lib/libpam_rssh.so";
        settings = {
          authorized_keys_command = "/etc/ssh/authorized_keys_command_kanidm";
          authorized_keys_command_user = "nobody";
--- a/cluster/services/idm/server.nix
+++ b/cluster/services/idm/server.nix
@ -33,9 +33,6 @@ in
      ldapbindaddress = "${ldapLink.ipv4}:${ldapLink.portStr}";
      origin = frontendLink.url;
      inherit domain;
-      online_backup = {
-        versions = 7;
-      };
    };
  };

--- a/cluster/services/incandescence/default.nix
+++ b/cluster/services/incandescence/default.nix
@ -3,7 +3,6 @@
 {
  imports = [
    ./options.nix
-    ./simulacrum/test-data.nix
  ];

  services.incandescence = {
@ -16,10 +15,6 @@
        ./provider-options.nix
      ];
    };
-    simulacrum = {
-      enable = true;
-      deps = [ "consul" "locksmith" ];
-      settings = ./simulacrum/test.nix;
-    };
+    simulacrum.deps = [ "consul" ];
  };
 }
--- a/cluster/services/incandescence/provider.nix
+++ b/cluster/services/incandescence/provider.nix
@ -1,7 +1,7 @@
 { cluster, config, lib, ... }:

 let
-  inherit (lib) concatStringsSep escapeShellArg flatten filter filterAttrs length mapAttrs mapAttrs' mapAttrsToList mkIf mkMerge optionalString pipe stringToCharacters;
+  inherit (lib) concatStringsSep escapeShellArg flatten filter filterAttrs length mapAttrs mapAttrs' mapAttrsToList mkIf mkMerge pipe stringToCharacters;

  cfg = config.services.incandescence;
  clusterCfg = cluster.config.incandescence;
@ -39,9 +39,6 @@ in
              fi
            ''))
            (concatStringsSep "\n")
-            (script: if script == "" then ''
-              echo "Nothing to create"
-            '' else script)
          ];
        };
        "ignite-${provider}-${formula}-change" = mkIf (formulaConfig.change != null) {
@ -61,16 +58,14 @@ in
              ) || echo "Change failed: ${object}"
            ''))
            (concatStringsSep "\n")
-            (script: if script == "" then ''
-              echo "Nothing to change"
-            '' else script)
          ];
        };
        "ignite-${provider}-${formula}-destroy" = {
          description = "Ignite Destruction: ${provider} - ${formula}";
          wantedBy = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
          before = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
-          inherit (providerConfig) wants after;
+          wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-change.service" ];
+          after = providerConfig.after ++ [ "ignite-${provider}-${formula}-change.service" ];
          serviceConfig.Type = "oneshot";
          distributed.enable = true;
          path = [ config.services.consul.package ] ++ providerConfig.packages;
@ -82,15 +77,13 @@ in
              (builtins.add 2)
              toString
            ];
-            needsFilter = clusterCfg.providers.${provider}.objects.${formula} != [];
            keyFilter = pipe clusterCfg.providers.${provider}.objects.${formula} [
              (map (x: escapeShellArg "^${x}$"))
              (concatStringsSep " \\\n  -e ")
            ];
            destroyAfterDays = toString formulaConfig.destroyAfterDays;
-            grep = optionalString needsFilter "grep -v -e ${keyFilter} |";
          in ''
-            consul kv get --keys ${kvRoot}/ | cut -d/ -f${fieldNum} | ${grep} while read object; do
+            consul kv get --keys ${kvRoot}/ | cut -d/ -f${fieldNum} | grep -v -e ${keyFilter} | while read object; do
              if consul kv get ${kvRoot}/$object/alive >/dev/null; then
                destroyOn="$(consul kv get ${kvRoot}/$object/destroyOn || true)"
                if [[ -z "$destroyOn" && "${destroyAfterDays}" -ne 0 ]]; then
--- a/cluster/services/incandescence/simulacrum/test-data.nix
+++ b/cluster/services/incandescence/simulacrum/test-data.nix
@ -1,8 +0,0 @@
-{ config, lib, ... }:
-{
-  incandescence = lib.mkIf config.simulacrum {
-    providers = config.lib.forService "incandescence" {
-      test.objects.example = [ "example1" "example2" ];
-    };
-  };
-}
--- a/cluster/services/incandescence/simulacrum/test.nix
+++ b/cluster/services/incandescence/simulacrum/test.nix
@ -1,47 +0,0 @@
-{ cluster, lib, ... }:
-
-let
-  providers = lib.take 2 cluster.config.services.incandescence.nodes.provider;
-in
-
-{
-  nodes = lib.genAttrs providers (lib.const {
-    services.incandescence.providers.test = {
-      wantedBy = [ "multi-user.target" ];
-      partOf = [ ];
-      formulae.example = {
-        create = x: "consul kv put testData/${x} ${x}";
-        destroy = "consul kv delete testData/$OBJECT";
-      };
-    };
-  });
-
-  testScript = ''
-    import json
-    nodeNames = json.loads('${builtins.toJSON providers}')
-    nodes = [ n for n in machines if n.name in nodeNames ]
-
-    start_all()
-
-    consulConfig = json.loads(nodes[0].succeed("cat /etc/consul.json"))
-    addr = consulConfig["addresses"]["http"]
-    port = consulConfig["ports"]["http"]
-    setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
-
-    with subtest("should create objects"):
-      for node in nodes:
-        node.wait_for_unit("incandescence-test.target")
-      nodes[0].succeed(f"[[ $({setEnv} consul kv get testData/example1) == example1 ]]")
-      nodes[0].succeed(f"[[ $({setEnv} consul kv get testData/example2) == example2 ]]")
-
-    with subtest("should destroy objects"):
-      nodes[0].succeed(f"{setEnv} consul kv put testData/example3 example3")
-      nodes[0].succeed(f"{setEnv} consul kv put services/incandescence/providers/test/formulae/example/example3/alive true")
-      nodes[1].succeed(f"{setEnv} consul kv get testData/example3")
-      for node in nodes:
-        node.systemctl("isolate default")
-      for node in nodes:
-        node.wait_for_unit("incandescence-test.target")
-      nodes[0].fail(f"{setEnv} consul kv get testData/example3")
-  '';
-}
--- a/cluster/services/ipfs/default.nix
+++ b/cluster/services/ipfs/default.nix
@ -29,7 +29,10 @@
      io-tweaks = [ "VEGAS" ];
      remote-api = [ "VEGAS" ];
    };
-    meshLinks.gateway.ipfsGateway.link.protocol = "http";
+    meshLinks.gateway = {
+      name = "ipfsGateway";
+      link.protocol = "http";
+    };
    nixos = {
      node = [
        ./node.nix
--- a/cluster/services/ipfs/monitoring.nix
+++ b/cluster/services/ipfs/monitoring.nix
@ -9,7 +9,7 @@ in
    environment = {
      OTEL_TRACES_EXPORTER = "otlp";
      OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-      OTEL_EXPORTER_OTLP_ENDPOINT = "${cluster.config.ways.ingest-traces-otlp.url}:443";
+      OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
      OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
      OTEL_TRACES_SAMPLER_ARG = "0.50";
    };
--- a/cluster/services/monitoring/client.nix
+++ b/cluster/services/monitoring/client.nix
@ -26,7 +26,7 @@ in {
        name = "logging";
        positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
        clients = singleton {
-          url = "${cluster.config.ways.ingest-logs.url}/loki/api/v1/push";
+          url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
        };
        scrape_configs = singleton {
          job_name = "journal";
--- a/cluster/services/monitoring/default.nix
+++ b/cluster/services/monitoring/default.nix
@ -18,6 +18,26 @@ in
      protocol = "http";
      ipv4 = meshIpFor "server";
    };
+    tempo = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
+    tempo-grpc = {
+      protocol = "http";
+      ipv4 = "127.0.0.1";
+    };
+    tempo-otlp-http = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
+    tempo-otlp-grpc = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
+    tempo-zipkin-http = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
  };
  hostLinks = lib.genAttrs config.services.monitoring.nodes.grafana (name: {
    grafana = {
@ -31,7 +51,6 @@ in
      blackbox = [ "checkmate" "grail" "prophet" ];
      grafana = [ "VEGAS" "prophet" ];
      logging = [ "VEGAS" "grail" ];
-      tracing = [ "VEGAS" "grail" ];
      server = [ "VEGAS" ];
    };
    nixos = {
@ -42,19 +61,14 @@ in
        ./provisioning/dashboards.nix
      ];
      logging = ./logging.nix;
-      tracing = ./tracing.nix;
      server = [
        ./server.nix
+        ./tracing.nix
      ];
    };
-    meshLinks = {
-      logging.loki.link.protocol = "http";
-      tracing = {
-        tempo.link.protocol = "http";
-        tempo-otlp-http.link.protocol = "http";
-        tempo-otlp-grpc.link.protocol = "grpc";
-        tempo-zipkin-http.link.protocol = "http";
-      };
+    meshLinks.logging = {
+      name = "loki";
+      link.protocol = "http";
    };
  };

@ -68,51 +82,29 @@ in
        nodes = config.services.monitoring.nodes.logging;
        format = "envFile";
      };
-      tempo-ingest.locksmith = {
-        nodes = config.services.monitoring.nodes.tracing;
-        format = "envFile";
-      };
-      tempo-query.locksmith = {
-        nodes = config.services.monitoring.nodes.tracing;
-        format = "envFile";
-      };
+      tempo = { };
    };
    buckets = {
      loki-chunks.allow = {
        loki-ingest = [ "read" "write" ];
        loki-query = [ "read" ];
      };
-      tempo-chunks.allow = {
-        tempo-ingest = [ "read" "write" ];
-        tempo-query = [ "read" ];
-      };
+      tempo-chunks.allow.tempo = [ "read" "write" ];
    };
  };

-  ways = let
-    query = consulService: {
-      inherit consulService;
-      internal = true;
-      extras.extraConfig = ''
-        proxy_read_timeout 3600s;
-      '';
+  ways = {
+    monitoring = {
+      consulService = "grafana";
+      extras.locations."/".proxyWebsockets = true;
    };
-    ingest = consulService: {
-      inherit consulService;
+    monitoring-logs = {
      internal = true;
+      consulService = "loki";
      extras.extraConfig = ''
        client_max_body_size 4G;
        proxy_read_timeout 3600s;
      '';
    };
-  in config.lib.forService "monitoring" {
-    monitoring = {
-      consulService = "grafana";
-      extras.locations."/".proxyWebsockets = true;
-    };
-    monitoring-logs = query "loki";
-    monitoring-traces = query "tempo";
-    ingest-logs = ingest "loki";
-    ingest-traces-otlp = ingest "tempo-ingest-otlp-grpc" // { grpc = true; };
  };
 }
--- a/cluster/services/monitoring/grafana-ha.nix
+++ b/cluster/services/monitoring/grafana-ha.nix
@ -73,16 +73,6 @@ in
          inherit (cluster.config.ways.monitoring-logs) url;
          type = "loki";
        }
-        {
-          name = "Tempo";
-          uid = "P214B5B846CF3925F";
-          inherit (cluster.config.ways.monitoring-traces) url;
-          type = "tempo";
-          jsonData = {
-            serviceMap.datasourceUid = "PBFA97CFB590B2093";
-            nodeGraph.enabled = true;
-          };
-        }
      ];
    };
  };
--- a/cluster/services/monitoring/provisioning/objects/dashboards/dashboard-dc7545ef-3180-4a5e-a289-1e64571ebb87.json
+++ b/cluster/services/monitoring/provisioning/objects/dashboards/dashboard-dc7545ef-3180-4a5e-a289-1e64571ebb87.json
@ -19,23 +19,10 @@
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 0,
-    "id": 1210,
+    "id": 16,
    "links": [],
    "liveNow": false,
    "panels": [
-      {
-        "collapsed": false,
-        "gridPos": {
-          "h": 1,
-          "w": 24,
-          "x": 0,
-          "y": 0
-        },
-        "id": 7,
-        "panels": [],
-        "title": "Replication",
-        "type": "row"
-      },
      {
        "datasource": {
          "type": "prometheus",
@ -71,7 +58,7 @@
          "h": 8,
          "w": 12,
          "x": 0,
-          "y": 1
+          "y": 0
        },
        "id": 2,
        "options": {
@ -79,7 +66,6 @@
          "graphMode": "area",
          "justifyMode": "auto",
          "orientation": "auto",
-          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [
              "lastNotNull"
@ -87,11 +73,9 @@
            "fields": "",
            "values": false
          },
-          "showPercentChange": false,
-          "textMode": "auto",
-          "wideLayout": true
+          "textMode": "auto"
        },
-        "pluginVersion": "11.1.3",
+        "pluginVersion": "9.5.1",
        "targets": [
          {
            "datasource": {
@ -154,7 +138,7 @@
          "h": 8,
          "w": 4,
          "x": 12,
-          "y": 1
+          "y": 0
        },
        "id": 3,
        "options": {
@ -162,17 +146,14 @@
          "graphMode": "area",
          "justifyMode": "auto",
          "orientation": "auto",
-          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [],
            "fields": "/^instance$/",
            "values": true
          },
-          "showPercentChange": false,
-          "textMode": "auto",
-          "wideLayout": true
+          "textMode": "auto"
        },
-        "pluginVersion": "11.1.3",
+        "pluginVersion": "9.5.1",
        "targets": [
          {
            "datasource": {
@ -217,7 +198,7 @@
          "h": 8,
          "w": 8,
          "x": 16,
-          "y": 1
+          "y": 0
        },
        "id": 5,
        "options": {
@ -273,7 +254,6 @@
              "mode": "palette-classic"
            },
            "custom": {
-              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
@ -287,7 +267,6 @@
                "tooltip": false,
                "viz": false
              },
-              "insertNulls": false,
              "lineInterpolation": "smooth",
              "lineStyle": {
                "fill": "solid"
@ -325,7 +304,7 @@
          "h": 13,
          "w": 12,
          "x": 0,
-          "y": 9
+          "y": 8
        },
        "id": 1,
        "options": {
@ -369,7 +348,6 @@
              "mode": "palette-classic"
            },
            "custom": {
-              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
@ -383,7 +361,6 @@
                "tooltip": false,
                "viz": false
              },
-              "insertNulls": false,
              "lineInterpolation": "smooth",
              "lineWidth": 1,
              "pointSize": 5,
@ -421,7 +398,7 @@
          "h": 13,
          "w": 12,
          "x": 12,
-          "y": 9
+          "y": 8
        },
        "id": 4,
        "options": {
@ -451,326 +428,24 @@
        ],
        "title": "Activity",
        "type": "timeseries"
-      },
-      {
-        "collapsed": false,
-        "gridPos": {
-          "h": 1,
-          "w": 24,
-          "x": 0,
-          "y": 22
-        },
-        "id": 8,
-        "panels": [],
-        "title": "Storage",
-        "type": "row"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              }
-            },
-            "mappings": [],
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 8,
-          "x": 0,
-          "y": 23
-        },
-        "id": 6,
-        "options": {
-          "displayLabels": [
-            "percent"
-          ],
-          "legend": {
-            "calcs": [],
-            "displayMode": "table",
-            "placement": "right",
-            "showLegend": true,
-            "sortBy": "Value",
-            "sortDesc": true,
-            "values": [
-              "percent",
-              "value"
-            ]
-          },
-          "pieType": "donut",
-          "reduceOptions": {
-            "calcs": [
-              "lastNotNull"
-            ],
-            "fields": "",
-            "values": false
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "pluginVersion": "11.1.3",
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "max by(datname) (sum by(datname, instance) (pg_database_size_bytes))",
-            "fullMetaSearch": false,
-            "includeNullMetadata": false,
-            "instant": true,
-            "legendFormat": "__auto",
-            "range": false,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "Database Size",
-        "type": "piechart"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisBorderShow": false,
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 25,
-              "gradientMode": "opacity",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "insertNulls": false,
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            },
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 5,
-          "x": 8,
-          "y": 23
-        },
-        "id": 9,
-        "options": {
-          "legend": {
-            "calcs": [
-              "lastNotNull",
-              "max"
-            ],
-            "displayMode": "table",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "pluginVersion": "11.1.3",
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "pg_wal_size_bytes",
-            "format": "time_series",
-            "fullMetaSearch": false,
-            "includeNullMetadata": true,
-            "instant": false,
-            "legendFormat": "{{instance}}",
-            "range": true,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "WAL Size",
-        "type": "timeseries"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "thresholds",
-              "seriesBy": "last"
-            },
-            "custom": {
-              "axisBorderShow": false,
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 25,
-              "gradientMode": "opacity",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "insertNulls": false,
-              "lineInterpolation": "linear",
-              "lineStyle": {
-                "fill": "solid"
-              },
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "decimals": 3,
-            "fieldMinMax": false,
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                }
-              ]
-            },
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 11,
-          "x": 13,
-          "y": 23
-        },
-        "id": 10,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": false
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "max(sum by(instance) (pg_database_size_bytes))",
-            "fullMetaSearch": false,
-            "includeNullMetadata": false,
-            "instant": false,
-            "legendFormat": "__auto",
-            "range": true,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "Total DB Size",
-        "type": "timeseries"
      }
    ],
    "refresh": "5m",
-    "schemaVersion": 39,
+    "schemaVersion": 38,
+    "style": "dark",
    "tags": [],
    "templating": {
      "list": []
    },
    "time": {
-      "from": "now-24h",
+      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {},
    "timezone": "",
    "title": "PostgreSQL HA",
    "uid": "dc7545ef-3180-4a5e-a289-1e64571ebb87",
-    "version": 2,
+    "version": 7,
    "weekStart": ""
  },
  "folderId": 0,
--- a/cluster/services/monitoring/secrets/tempo-secrets.age
+++ b/cluster/services/monitoring/secrets/tempo-secrets.age
@ -0,0 +1,16 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A KhCGp7PAMGrEdzRxBrsW4tRk30JwpI+4lPzrRCUhSw4
+8s7WqA5c3zS1euN5R+jfFNBdvr8OQW8P4NFeqtNsIKo
+-> ssh-ed25519 5/zT0w 79hJQ2H76EZTW7YcQFCtKaS5Kbssx4Z8dPFjIVzRgFk
+A1fDJbUnyIRy+kWa3PhJNj/SdRPlcEy6FYsAfnuZ2AQ
+-> ssh-ed25519 d3WGuA aylkdL1KliM1NfrYDGlG8X6YjXvVUCU4sV90I+a840U
+6sXdqIPjtoNSylZRh1DCghHOwDo+fC7WB4QWQoWmG48
+-> //gd+2-grease baUWA$3 z-qs3W O/2.1W
+Sfq3+rkMJhpUTTmcos5TaaUtX2Ip9pciHAZLiWPix+C9N7ccac/1W5RNedMJCLsq
+MQ+xKzexf8+hgNVhKOksvbKBBROXqk1bUOKk8w3OgFPmmByzmCBUwkdkeu5DFTYR
+rg
+--- kUl1uIPRkM5y7C68kdN22pMKXP7gazyha4PE+ap0Jqw
+w>×Àè¥
+<15>CÈ,\‰ßœI¯ˆúHxG@^Çá“PåÃþÙÏlw6µŽ{þ’rb‘é5æ†T>Êñ
+ÚWÜ¤X4Kp(ß?9ˆß^^oP3f </v3N$ê¤sÓbŽ¾>O™÷œ+òN0Î¬ïµàDtêŽ5Vº#è ¶³‘Uã îŸ#y|›@ŒGzSi»ô*·HùüŽ]
+êŽ€5›
--- a/cluster/services/monitoring/tracing.nix
+++ b/cluster/services/monitoring/tracing.nix
@ -1,16 +1,14 @@
 { cluster, config, pkgs, ... }:

 let
-  inherit (cluster.config.links) prometheus-ingest;
-  inherit (config.links) tempo-grpc;
-  links = cluster.config.hostLinks.${config.networking.hostName};
+  inherit (cluster.config) links;
  dataDir = "/srv/storage/private/tempo";
  tempoConfig = {
    server = {
      http_listen_address = links.tempo.ipv4;
      http_listen_port = links.tempo.port;
-      grpc_listen_address = tempo-grpc.ipv4;
-      grpc_listen_port = tempo-grpc.port;
+      grpc_listen_address = links.tempo-grpc.ipv4;
+      grpc_listen_port = links.tempo-grpc.port;
    };
    distributor.receivers = {
      otlp = {
@ -21,7 +19,7 @@ let
      };
      zipkin.endpoint = links.tempo-zipkin-http.tuple;
    };
-    querier.frontend_worker.frontend_address = tempo-grpc.tuple;
+    querier.frontend_worker.frontend_address = links.tempo-grpc.tuple;
    ingester = {
      trace_idle_period = "30s";
      max_block_bytes = 1000000;
@ -58,7 +56,7 @@ let
        path = "${dataDir}/generator/wal";
        remote_write = [
          {
-            url = "${prometheus-ingest.url}/api/v1/write";
+            url = "${links.prometheus-ingest.url}/api/v1/write";
            send_exemplars = true;
          }
        ];
@ -70,11 +68,7 @@ let
    ];
  };
 in {
-  links.tempo-grpc.protocol = "http";
-
-  services.locksmith.waitForSecrets.tempo = [
-    "garage-tempo-ingest"
-  ];
+  age.secrets.tempoSecrets.file = ./secrets/tempo-secrets.age;

  users.users.tempo = {
    isSystemUser = true;
@ -87,53 +81,24 @@ in {

  systemd.services.tempo = {
    wantedBy = [ "multi-user.target" ];
-    distributed = {
-      enable = true;
-      registerServices = [
-        "tempo"
-        "tempo-ingest-otlp-grpc"
-      ];
-    };
    serviceConfig = {
      User = "tempo";
      Group = "tempo";
      ExecStart = "${pkgs.tempo}/bin/tempo -config.file=${pkgs.writeText "tempo.yaml" (builtins.toJSON tempoConfig)}";
      PrivateTmp = true;
-      EnvironmentFile = "/run/locksmith/garage-tempo-ingest";
+      EnvironmentFile = config.age.secrets.tempoSecrets.path;
    };
  };
-
-  consul.services = {
-    tempo = {
-      mode = "manual";
-      definition = {
-        name = "tempo";
-        address = links.tempo.ipv4;
-        inherit (links.tempo) port;
-        checks = [
-          {
-            name = "Tempo";
-            id = "service:tempo:backend";
-            interval = "5s";
-            http = "${links.tempo.url}/ready";
-          }
-        ];
+  services.grafana.provision.datasources.settings.datasources = [
+    {
+      name = "Tempo";
+      uid = "P214B5B846CF3925F";
+      inherit (links.tempo) url;
+      type = "tempo";
+      jsonData = {
+        serviceMap.datasourceUid = "PBFA97CFB590B2093"; # prometheus
+        nodeGraph.enabled = true;
      };
-    };
-    tempo-ingest-otlp-grpc = {
-      mode = "manual";
-      definition = {
-        name = "tempo-ingest-otlp-grpc";
-        address = links.tempo-otlp-grpc.ipv4;
-        inherit (links.tempo-otlp-grpc) port;
-        checks = [
-          {
-            name = "Tempo Service Status";
-            id = "service:tempo-ingest-otlp-grpc:tempo";
-            alias_service = "tempo";
-          }
-        ];
-      };
-    };
-  };
+    }
+  ];
 }
--- a/cluster/services/nextcloud/host.nix
+++ b/cluster/services/nextcloud/host.nix
@ -18,7 +18,7 @@ in
    };
  };
  services.nextcloud = {
-    package = pkgs.nextcloud30;
+    package = pkgs.nextcloud29;
    enable = true;
    https = true;
    hostName = "storage.${depot.lib.meta.domain}";
--- a/cluster/services/patroni/create-databases.nix
+++ b/cluster/services/patroni/create-databases.nix
@ -49,9 +49,7 @@ in
      user = {
        destroyAfterDays = 0;
        create = user: psqlSecret "${genPassword} ${user}" ''
-          SELECT 'CREATE USER ${user}'
-          WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${user}')\gexec
-          ALTER USER ${user} PASSWORD '@SECRET@';
+          CREATE USER ${user} PASSWORD '@SECRET@';
        '';
        destroy = psqlSecret "printenv OBJECT" ''
          DROP USER @SECRET@;
@ -61,11 +59,7 @@ in
        destroyAfterDays = 30;
        deps = [ "user" ];
        create = db: psql ''
-          SELECT 'CREATE DATABASE ${db} OWNER ${cfg.databases.${db}.owner}'
-          WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${db}')\gexec
-        '';
-        change = db: psql ''
-          ALTER DATABASE ${db} OWNER TO ${cfg.databases.${db}.owner};
+          CREATE DATABASE ${db} OWNER ${cfg.databases.${db}.owner};
        '';
        destroy = psqlSecret "printenv OBJECT" ''
          DROP DATABASE @SECRET@;
--- a/cluster/services/patroni/default.nix
+++ b/cluster/services/patroni/default.nix
@ -4,7 +4,6 @@
  imports = [
    ./options.nix
    ./incandescence.nix
-    ./simulacrum/test-data.nix
  ];

  links = {
@ -37,10 +36,6 @@
      PATRONI_REWIND_PASSWORD = default;
      metricsCredentials.nodes = nodes.worker;
    };
-    simulacrum = {
-      enable = true;
-      deps = [ "consul" "incandescence" "locksmith" ];
-      settings = ./simulacrum/test.nix;
-    };
+    simulacrum.deps = [ "consul" "incandescence" "locksmith" ];
  };
 }
--- a/cluster/services/patroni/options.nix
+++ b/cluster/services/patroni/options.nix
@ -16,7 +16,6 @@ in
          };
        };
      }));
-      default = {};
    };
    users = mkOption {
      type = attrsOf (submodule ({ ... }: {
@ -33,7 +32,6 @@ in
          };
        };
      }));
-      default = {};
    };
  };
 }
--- a/cluster/services/patroni/simulacrum/test-data.nix
+++ b/cluster/services/patroni/simulacrum/test-data.nix
@ -1,19 +0,0 @@
-{ config, lib, ... }:
-{
-  patroni = lib.mkIf config.simulacrum {
-    databases = config.lib.forService "patroni" {
-      testdb.owner = "testuser";
-      existingdb.owner = "existinguser";
-    };
-    users = config.lib.forService "patroni" {
-      testuser.locksmith = {
-        nodes = config.services.patroni.nodes.haproxy;
-        format = "pgpass";
-      };
-      existinguser.locksmith = {
-        nodes = config.services.patroni.nodes.haproxy;
-        format = "pgpass";
-      };
-    };
-  };
-}
--- a/cluster/services/patroni/simulacrum/test.nix
+++ b/cluster/services/patroni/simulacrum/test.nix
@ -1,117 +0,0 @@
-{ cluster, ... }:
-
-let
-  clusterName = "poseidon";
-  link = cluster.config.links.patroni-pg-access;
-in
-{
-  defaults = { depot, pkgs, ... }: {
-    environment.systemPackages = [
-      pkgs.jq
-      depot.packages.postgresql
-    ];
-    services.patroni.settings.postgresql.pg_hba = [
-      "host postgres postgres 0.0.0.0/0 trust"
-    ];
-  };
-
-  # taken from https://github.com/phfroidmont/nixpkgs/blob/patroni-module/nixos/tests/patroni.nix
-  testScript = ''
-    import json
-    nodeNames = json.loads('${builtins.toJSON cluster.config.services.patroni.nodes.worker}')
-    clientNames = json.loads('${builtins.toJSON cluster.config.services.patroni.nodes.haproxy}')
-    nodes = [ n for n in machines if n.name in nodeNames ]
-    clients = [ n for n in machines if n.name in clientNames ]
-
-    def booted(nodes):
-      return filter(lambda node: node.booted, nodes)
-
-    def wait_for_all_nodes_ready(expected_replicas=2):
-        booted_nodes = booted(nodes)
-        for node in booted_nodes:
-            node.wait_for_unit("patroni.service")
-            print(node.succeed("patronictl list ${clusterName}"))
-            node.wait_until_succeeds(f"[ $(patronictl list -f json ${clusterName} | jq 'length') == {expected_replicas + 1} ]")
-            node.wait_until_succeeds("[ $(patronictl list -f json ${clusterName} | jq 'map(select(.Role | test(\"^Leader$\"))) | map(select(.State | test(\"^running$\"))) | length') == 1 ]")
-            node.wait_until_succeeds(f"[ $(patronictl list -f json ${clusterName} | jq 'map(select(.Role | test(\"^Replica$\"))) | map(select(.State | test(\"^streaming$\"))) | length') == {expected_replicas} ]")
-            print(node.succeed("patronictl list ${clusterName}"))
-        for client in booted(clients):
-            client.wait_until_succeeds("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --command='select 1;'")
-
-    def run_dummy_queries():
-        for client in booted(clients):
-            client.succeed("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='insert into dummy(val) values (101);'")
-            client.succeed("test $(psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='select val from dummy where val = 101;') -eq 101")
-            client.succeed("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='delete from dummy where val = 101;'")
-
-    start_all()
-
-    with subtest("should bootstrap a new patroni cluster"):
-        wait_for_all_nodes_ready()
-
-    with subtest("should be able to insert and select"):
-        booted_clients = list(booted(clients))
-        booted_clients[0].succeed("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --command='create table dummy as select * from generate_series(1, 100) as val;'")
-        for client in booted_clients:
-            client.succeed("test $(psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100")
-
-    with subtest("should restart after all nodes are crashed"):
-        for node in nodes:
-            node.crash()
-        for node in nodes:
-            node.start()
-        wait_for_all_nodes_ready()
-
-    with subtest("should be able to run queries while any one node is crashed"):
-        masterNodeName = nodes[0].succeed("patronictl list -f json ${clusterName} | jq '.[] | select(.Role | test(\"^Leader$\")) | .Member' -r").strip()
-        masterNodeIndex = next((i for i, v in enumerate(nodes) if v.name == masterNodeName))
-
-        # Move master node at the end of the list to avoid multiple failovers (makes the test faster and more consistent)
-        nodes.append(nodes.pop(masterNodeIndex))
-
-        for node in nodes:
-            node.crash()
-            wait_for_all_nodes_ready(1)
-
-            # Execute some queries while a node is down.
-            run_dummy_queries()
-
-            # Restart crashed node.
-            node.start()
-            wait_for_all_nodes_ready()
-
-            # Execute some queries with the node back up.
-            run_dummy_queries()
-
-    with subtest("should create databases and users via incandescence"):
-        for client in clients:
-            client.succeed(f"PGPASSFILE=/run/locksmith/patroni-testuser psql -h ${link.ipv4} -p ${link.portStr} -U testuser -d testdb --command='create table test_table_{client.name} as select * from generate_series(1, 10) as val;'")
-            client.fail("PGPASSFILE=/run/locksmith/patroni-testuser psql -h ${link.ipv4} -p ${link.portStr} -U testuser -d postgres --command='select * from dummy;'")
-
-    with subtest("should take over existing databases and users via incandescence"):
-        for cmd in [
-            "drop database existingdb;",
-            "drop user existinguser;",
-            "create database existingdb owner postgres;",
-            "create user existinguser;"
-        ]:
-            clients[0].succeed(f"psql -h ${link.ipv4} -p ${link.portStr} -U postgres --command='{cmd}'")
-
-        for client in clients:
-            client.fail(f"PGPASSFILE=/run/locksmith/patroni-existinguser psql -h ${link.ipv4} -p ${link.portStr} -U existinguser -d existingdb --command='create table test_table_{client.name} as select * from generate_series(1, 10) as val;'")
-
-        consulConfig = json.loads(clients[0].succeed("cat /etc/consul.json"))
-        addr = consulConfig["addresses"]["http"]
-        port = consulConfig["ports"]["http"]
-        setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
-        clients[0].succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/patroni/formulae/database/existingdb")
-        clients[0].succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/patroni/formulae/user/existinguser")
-
-        for node in nodes:
-            node.systemctl("restart incandescence-patroni.target")
-        clients[0].succeed("[[ $(psql -h ${link.ipv4} -p ${link.portStr} -U postgres --tuples-only --csv --command=\"SELECT pg_roles.rolname FROM pg_database JOIN pg_roles ON pg_database.datdba = pg_roles.oid WHERE pg_database.datname = 'existingdb'\") == existinguser ]]")
-        for client in clients:
-            client.succeed(f"PGPASSFILE=/run/locksmith/patroni-existinguser psql -h ${link.ipv4} -p ${link.portStr} -U existinguser -d existingdb --command='create table test_table_{client.name} as select * from generate_series(1, 10) as val;'")
-            client.fail("PGPASSFILE=/run/locksmith/patroni-existinguser psql -h ${link.ipv4} -p ${link.portStr} -U existinguser -d postgres --command='select * from dummy;'")
-  '';
-}
--- a/cluster/services/patroni/worker.nix
+++ b/cluster/services/patroni/worker.nix
@ -25,10 +25,6 @@ in
    "d '${baseDir}' 0700 patroni patroni - -"
    "d '${walDir}' 0700 patroni patroni - -"
  ];
-  systemd.services.patroni = {
-    requires = [ "consul-ready.service" ];
-    after = [ "consul-ready.service" ];
-  };
  services.patroni = {
    enable = true;
    name = hostName;
@ -61,7 +57,6 @@ in
        };
        use_pg_rewind = true;
        use_slots = true;
-        synchronous_mode = true;
        authentication = {
          replication.username = "patronirep";
          rewind.username = "patronirew";
@ -72,8 +67,6 @@ in
          wal_level = "replica";
          hot_standby_feedback = "on";
          unix_socket_directories = "/tmp";
-          synchronous_commit = "on";
-          wal_keep_size = 2048;
        };
        pg_hba = [
          "host replication patronirep ${net} scram-sha-256"
--- a/cluster/services/sso/default.nix
+++ b/cluster/services/sso/default.nix
@ -1,4 +1,4 @@
-{ config, depot, ... }:
+{ depot, ... }:

 {
  services.sso = {
@ -18,12 +18,4 @@
    login.target = ssoAddr;
    account.target = ssoAddr;
  };
-
-  patroni = config.lib.forService "sso" {
-    databases.keycloak = {};
-    users.keycloak.locksmith = {
-      nodes = config.services.sso.nodes.host;
-      format = "raw";
-    };
-  };
 }
--- a/cluster/services/sso/host.nix
+++ b/cluster/services/sso/host.nix
@ -8,10 +8,12 @@ in
 {
  links.keycloak.protocol = "http";

-  services.locksmith.waitForSecrets.keycloak = [
-    "patroni-keycloak"
-  ];
-
+  age.secrets.keycloak-dbpass = {
+    file = ../../../secrets/keycloak-dbpass.age;
+    owner = "root";
+    group = "root";
+    mode = "0400";
+  };
  services.nginx.virtualHosts = { 
    "${login}" = lib.recursiveUpdate (vhosts.proxy kc.url) {
      locations = {
@ -34,14 +36,13 @@ in
      host = patroni.ipv4;
      inherit (patroni) port;
      useSSL = false;
-      passwordFile = "/run/locksmith/patroni-keycloak";
+      passwordFile = config.age.secrets.keycloak-dbpass.path;
    };
    settings = {
-      http-enabled = true;
      http-host = kc.ipv4;
      http-port = kc.port;
      hostname = login;
-      proxy-headers = "xforwarded";
+      proxy = "edge";
      # for backcompat, TODO: remove
      http-relative-path = "/auth";
    };
@ -53,7 +54,7 @@ in
      "-Dotel.traces.exporter=otlp"
    ];
    OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.ways.ingest-traces-otlp.url;
+    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
    OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
    OTEL_TRACES_SAMPLER_ARG = "0.50";
  };
--- a/cluster/services/storage/default.nix
+++ b/cluster/services/storage/default.nix
@ -39,6 +39,7 @@ in
        ./garage.nix
        ./garage-options.nix
        ./garage-layout.nix
+        ./incandescence-ascensions.nix
      ] ++ lib.optionals config.simulacrum [
        ./simulacrum/snakeoil-rpc-secret.nix
      ];
@ -56,8 +57,8 @@ in
    };
    simulacrum = {
      enable = true;
-      deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" "ways" ];
-      settings = ./simulacrum/test.nix;
+      deps = [ "consul" "locksmith" "incandescence" "patroni" "ways" ];
+      settings = ./test.nix;
    };
  };

--- a/cluster/services/storage/garage-gateway.nix
+++ b/cluster/services/storage/garage-gateway.nix
@ -36,9 +36,10 @@ in
        inherit (linkWeb) port;
        checks = [
          {
-            name = "Garage Service Status";
-            id = "service:garage-web:garage";
-            alias_service = "garage";
+            name = "Garage Node";
+            id = "service:garage-web:node";
+            interval = "5s";
+            http = "${config.links.garageMetrics.url}/health";
          }
        ];
      };
--- a/cluster/services/storage/garage-options.nix
+++ b/cluster/services/storage/garage-options.nix
@ -179,13 +179,9 @@ in
        key = {
          destroyAfterDays = 0;
          create = key: ''
-            if [[ "$(garage key info ${lib.escapeShellArg key} 2>&1 >/dev/null)" == "Error: 0 matching keys" ]]; then
-              # don't print secret key
-              garage key new --name ${lib.escapeShellArg key} >/dev/null
-              echo Key ${lib.escapeShellArg key} was created.
-            else
-              echo "Key already exists, assuming ownership"
-            fi
+            # don't print secret key
+            garage key new --name ${lib.escapeShellArg key} >/dev/null
+            echo Key ${lib.escapeShellArg key} was created.
          '';
          destroy = ''
            garage key delete --yes "$OBJECT"
@ -200,11 +196,7 @@ in
          deps = [ "key" ];
          destroyAfterDays = 30;
          create = bucket: ''
-            if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == "Error: Bucket not found" ]]; then
-              garage bucket create ${lib.escapeShellArg bucket}
-            else
-              echo "Bucket already exists, assuming ownership"
-            fi
+            garage bucket create ${lib.escapeShellArg bucket}
          '';
          destroy = ''
            garage bucket delete --yes "$OBJECT"
--- a/cluster/services/storage/incandescence-ascensions.nix
+++ b/cluster/services/storage/incandescence-ascensions.nix
@ -0,0 +1,18 @@
+{ config, lib, ... }:
+
+{
+  system.ascensions = {
+    incandescence-garage = lib.mkIf (config.services.incandescence.providers ? garage) {
+      incantations = i: [
+        (i.runGarage /*bash*/ ''
+          garage bucket list | tail -n +2 | cut -d' ' -f3 | while read bucket; do
+            ${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/bucket/$1/alive" true''} "$bucket"
+          done
+          garage key list | tail -n +2 | cut -d' ' -f5 | while read key; do
+            ${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/key/$1/alive" true''} "$key"
+          done
+        '')
+      ];
+    };
+  };
+}
--- a/cluster/services/storage/simulacrum/test.nix
+++ b/cluster/services/storage/simulacrum/test.nix
--- a/cluster/services/ways/default.nix
+++ b/cluster/services/ways/default.nix
@ -3,17 +3,12 @@
 {
  imports = [
    ./options
-    ./simulacrum/test-data.nix
  ];

  services.ways = {
    nodes.host = config.services.websites.nodes.host;
    nixos.host = ./host.nix;
-    simulacrum = {
-      enable = true;
-      deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
-      settings = ./simulacrum/test.nix;
-    };
+    simulacrum.deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
  };

  dns.records = lib.mapAttrs'
--- a/cluster/services/ways/host.nix
+++ b/cluster/services/ways/host.nix
@ -1,15 +1,11 @@
-{ cluster, config, depot, lib, pkgs, ... }:
+{ cluster, config, lib, pkgs, ... }:

 let
  externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;

  internalWays = lib.filterAttrs (_: cfg: cfg.internal) cluster.config.ways;

-  byMode = lib.pipe cluster.config.ways [
-    (lib.attrsToList)
-    (lib.groupBy (way: way.value.mode))
-    (lib.mapAttrs (n: v: lib.listToAttrs v))
-  ];
+  consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
 in

 {
@ -29,20 +25,7 @@ in
            ];
            locations = lib.mkMerge [
              {
-                "/" = if cfg.mode == "static" then {
-                  root = cfg.static {
-                    inherit depot;
-                    inherit pkgs;
-                    inherit (pkgs) system;
-                  };
-                } else if cfg.grpc then {
-                  extraConfig = ''
-                    set $nix_proxy_grpc_target ${cfg.target};
-                    grpc_pass $nix_proxy_grpc_target;
-                  '';
-                } else {
-                  proxyPass = cfg.target;
-                };
+                "/".proxyPass = cfg.target;
                "${cfg.healthCheckPath}".extraConfig = "access_log off;";
              }
              {
@ -57,7 +40,7 @@ in
      };
    }) cluster.config.ways;

-    appendHttpConfig = lib.mkIf (byMode.consul != {}) ''
+    appendHttpConfig = lib.mkIf (consulServiceWays != {}) ''
      include /run/consul-template/nginx-ways-*.conf;
    '';
  };
@ -77,7 +60,7 @@ in
    value.distributed.enable = true;
  }) externalWays;

-  services.consul-template.instances.ways = lib.mkIf (byMode.consul != {}) {
+  services.consul-template.instances.ways = lib.mkIf (consulServiceWays != {}) {
    user = "nginx";
    group = "nginx";
    settings = {
@ -96,14 +79,14 @@ in
              {{ else }}
              # upstream ${cfg.nginxUpstreamName} (${cfg.consulService}): no servers available
              {{ end }}
-            '') byMode.consul;
+            '') consulServiceWays;
          in pkgs.writeText "ways-upstreams.ctmpl" (lib.concatStringsSep "\n" (lib.unique upstreams));
          destination = "/run/consul-template/nginx-ways-upstreams.conf";
-          exec.command = lib.singleton (pkgs.writeShellScript "ways-reload" ''
-            if ${config.systemd.package}/bin/systemctl is-active nginx.service; then
-              exec ${config.services.nginx.package}/bin/nginx -s reload -g 'pid /run/nginx/nginx.pid;'
-            fi
-          '');
+          exec.command = [
+            "${config.services.nginx.package}/bin/nginx"
+            "-s" "reload"
+            "-g" "pid /run/nginx/nginx.pid;"
+          ];
        }
      ];
    };
--- a/cluster/services/ways/options/way.nix
+++ b/cluster/services/ways/options/way.nix
@ -35,12 +35,6 @@ with lib;
      };
    };

-    grpc = mkOption {
-      description = "Whether this endpoint is a gRPC service.";
-      type = types.bool;
-      default = false;
-    };
-
    target = mkOption {
      type = types.str;
    };
@ -58,10 +52,6 @@ with lib;
      type = types.str;
    };

-    static = mkOption {
-      type = with types; functionTo (coercedTo package (package: "${package.webroot or package}") str);
-    };
-
    healthCheckPath = mkOption {
      type = types.path;
      default = "/.well-known/ways/internal-health-check";
@ -73,10 +63,10 @@ with lib;
      default = "https://${name}.${config.domainSuffix}";
    };

-    mode = mkOption {
-      type = types.enum [ "simple" "consul" "static" ];
+    useConsul = mkOption {
+      type = types.bool;
      internal = true;
-      default = "simple";
+      default = false;
    };

    nginxUpstreamName = mkOption {
@ -109,15 +99,12 @@ with lib;

  config = lib.mkMerge [
    (lib.mkIf options.consulService.isDefined {
-      mode = "consul";
+      useConsul = true;
      nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
-      target = "${if config.grpc then "grpc" else "http"}://${options.nginxUpstreamName.value}";
+      target = "http://${options.nginxUpstreamName.value}";
    })
    (lib.mkIf options.bucket.isDefined {
      consulService = "garage-web";
    })
-    (lib.mkIf options.static.isDefined {
-      mode = "static";
-    })
  ];
 }
--- a/cluster/services/ways/simulacrum/test-data.nix
+++ b/cluster/services/ways/simulacrum/test-data.nix
@ -1,11 +0,0 @@
-{ config, lib, ... }:
-{
-  ways = lib.mkIf config.simulacrum {
-    ways-test-simple = config.lib.forService "ways" {
-      target = "http://nowhere";
-    };
-    ways-test-consul = config.lib.forService "ways" {
-      consulService = "ways-test-service";
-    };
-  };
-}
--- a/cluster/services/ways/simulacrum/test.nix
+++ b/cluster/services/ways/simulacrum/test.nix
@ -1,55 +0,0 @@
-{ cluster, config, lib, ... }:
-
-let
-  inherit (cluster._module.specialArgs.depot.lib.meta) domain;
-in
-
-{
-  nodes = lib.mkMerge [
-    {
-      nowhere = { pkgs, ... }: {
-        networking.firewall.allowedTCPPorts = [ 8080 ];
-        systemd.services.ways-simple-service = let
-          webroot = pkgs.writeTextDir "example.txt" "hello world";
-        in {
-          wantedBy = [ "multi-user.target" ];
-          serviceConfig = {
-            ExecStart = "${pkgs.darkhttpd}/bin/darkhttpd ${webroot} --port 8080";
-            DynamicUser = true;
-          };
-        };
-      };
-    }
-    (lib.genAttrs cluster.config.services.ways.nodes.host (lib.const {
-      services.nginx.upstreams.nowhere.servers = {
-        "${(builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address}:8080" = {};
-      };
-      consul.services.ways-test-service = {
-        unit = "consul";
-        mode = "external";
-        definition = {
-          name = "ways-test-service";
-          address = (builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address;
-          port = 8080;
-        };
-      };
-    }))
-  ];
-
-  testScript = ''
-    import json
-    nodeNames = json.loads('${builtins.toJSON cluster.config.services.ways.nodes.host}')
-    nodes = [ n for n in machines if n.name in nodeNames ]
-
-    start_all()
-    nowhere.wait_for_unit("multi-user.target")
-    for node in nodes:
-      node.wait_for_unit("multi-user.target")
-
-    with subtest("single-target service"):
-      nowhere.succeed("curl -f https://ways-test-simple.${domain}")
-
-    with subtest("consul-managed service"):
-      nowhere.succeed("curl -f https://ways-test-consul.${domain}")
-  '';
-}
--- a/cluster/services/wireguard/default.nix
+++ b/cluster/services/wireguard/default.nix
@ -93,9 +93,5 @@ in
      nodes = config.services.wireguard.nodes.mesh;
      shared = false;
    };
-    simulacrum = {
-      enable = true;
-      settings = ./test.nix;
-    };
  };
 }
--- a/cluster/services/wireguard/test.nix
+++ b/cluster/services/wireguard/test.nix
@ -1,28 +0,0 @@
-{ cluster, lib, ... }:
-
-{
-  defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
-
-  testScript = ''
-    start_all()
-    ${lib.pipe cluster.config.services.wireguard.nodes.mesh [
-      (map (node: /*python*/ ''
-        ${node}.wait_for_unit("wireguard-wgmesh.target")
-      ''))
-      (lib.concatStringsSep "\n")
-    ]}
-
-    ${lib.pipe cluster.config.services.wireguard.nodes.mesh [
-      (map (node: /*python*/ ''
-        with subtest("${node} can reach all other nodes"):
-          ${lib.pipe (cluster.config.services.wireguard.otherNodes.mesh node) [
-            (map (peer: /*python*/ ''
-              ${node}.succeed("ping -c3 ${cluster.config.hostLinks.${peer}.mesh.extra.meshIp}")
-            ''))
-            (lib.concatStringsSep "\n  ")
-          ]}
-      ''))
-      (lib.concatStringsSep "\n")
-    ]}
-  '';
-}
--- a/flake.lock
+++ b/flake.lock
@ -10,11 +10,11 @@
        "systems": "systems"
      },
      "locked": {
-        "lastModified": 1723293904,
-        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
+        "lastModified": 1722339003,
+        "narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
        "owner": "ryantm",
        "repo": "agenix",
-        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
+        "rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
        "type": "github"
      },
      "original": {
@ -29,8 +29,9 @@
        "flake-compat": [
          "blank"
        ],
-        "flake-parts": "flake-parts",
-        "nix-github-actions": "nix-github-actions",
+        "flake-utils": [
+          "repin-flake-utils"
+        ],
        "nixpkgs": [
          "nixpkgs"
        ],
@ -39,11 +40,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1730906442,
-        "narHash": "sha256-tBuyb8jWBSHHgcIrOfiyQJZGY1IviMzH2V74t7gWfgI=",
+        "lastModified": 1722472866,
+        "narHash": "sha256-GJIz4M5HDB948Ex/8cPvbkrNzl/eKUE7/c21JBu4lb8=",
        "owner": "zhaofengli",
        "repo": "attic",
-        "rev": "d0b66cf897e4d55f03d341562c9821dc4e566e54",
+        "rev": "e127acbf9a71ebc0c26bc8e28346822e0a6e16ba",
        "type": "github"
      },
      "original": {
@ -75,11 +76,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1722960479,
-        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "lastModified": 1717025063,
+        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
        "type": "github"
      },
      "original": {
@ -117,11 +118,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1728330715,
-        "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
+        "lastModified": 1722113426,
+        "narHash": "sha256-Yo/3loq572A8Su6aY5GP56knpuKYRvM2a1meP9oJZCw=",
        "owner": "numtide",
        "repo": "devshell",
-        "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
+        "rev": "67cce7359e4cd3c45296fb4aaf6a19e2a9c757ae",
        "type": "github"
      },
      "original": {
@ -173,11 +174,11 @@
    "flake-compat_2": {
      "flake": false,
      "locked": {
-        "lastModified": 1696426674,
-        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
        "owner": "edolstra",
        "repo": "flake-compat",
-        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
        "type": "github"
      },
      "original": {
@ -189,7 +190,6 @@
    "flake-parts": {
      "inputs": {
        "nixpkgs-lib": [
-          "attic",
          "nixpkgs"
        ]
      },
@ -210,15 +210,16 @@
    "flake-parts_2": {
      "inputs": {
        "nixpkgs-lib": [
+          "nix-super",
          "nixpkgs"
        ]
      },
      "locked": {
-        "lastModified": 1730504689,
-        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
        "owner": "hercules-ci",
        "repo": "flake-parts",
-        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
        "type": "github"
      },
      "original": {
@ -227,55 +228,18 @@
        "type": "github"
      }
    },
-    "flake-parts_3": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "nix-super",
-          "nixpkgs"
-        ]
-      },
+    "flake-utils": {
      "locked": {
-        "lastModified": 1719994518,
-        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
        "type": "github"
      },
      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "git-hooks-nix": {
-      "inputs": {
-        "flake-compat": [
-          "nix-super"
-        ],
-        "gitignore": [
-          "nix-super"
-        ],
-        "nixpkgs": [
-          "nix-super",
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": [
-          "nix-super",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1721042469,
-        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
-        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
+        "owner": "numtide",
+        "repo": "flake-utils",
        "type": "github"
      }
    },
@ -304,11 +268,11 @@
        "nixpkgs": "nixpkgs"
      },
      "locked": {
-        "lastModified": 1723736589,
-        "narHash": "sha256-/Vdg5ZKtP71ZEKVV6JXlrOEu0CM2Flcs+nwDmWRzgjQ=",
+        "lastModified": 1720223941,
+        "narHash": "sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4=",
        "owner": "hercules-ci",
        "repo": "hercules-ci-agent",
-        "rev": "c303cc8e437c0fd26b9452472e7df5aa374e9177",
+        "rev": "2e10fb21fc2e07edf40763b73443e5934bd40947",
        "type": "github"
      },
      "original": {
@ -327,11 +291,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1730903510,
-        "narHash": "sha256-mnynlrPeiW0nUQ8KGZHb3WyxAxA3Ye/BH8gMjdoKP6E=",
+        "lastModified": 1719226092,
+        "narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=",
        "owner": "hercules-ci",
        "repo": "hercules-ci-effects",
-        "rev": "b89ac4d66d618b915b1f0a408e2775fe3821d141",
+        "rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5",
        "type": "github"
      },
      "original": {
@ -385,16 +349,15 @@
    "libgit2": {
      "flake": false,
      "locked": {
-        "lastModified": 1715853528,
-        "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
+        "lastModified": 1697646580,
+        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
        "owner": "libgit2",
        "repo": "libgit2",
-        "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
+        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
        "type": "github"
      },
      "original": {
        "owner": "libgit2",
-        "ref": "v1.8.1",
        "repo": "libgit2",
        "type": "github"
      }
@ -436,11 +399,11 @@
        "systems": "systems_2"
      },
      "locked": {
-        "lastModified": 1725623828,
-        "narHash": "sha256-5Zrn72PO9yBaNO4Gd5uOsEmRpYH5rVAFKOQ5h2PxyhU=",
+        "lastModified": 1722409392,
+        "narHash": "sha256-8QuMS00EutmqzAIPxyJEPxM8EHiWlSKs6E2Htoh3Kes=",
        "owner": "numtide",
        "repo": "nar-serve",
-        "rev": "e5c749a444f2d14f381c75ef3a8feaa82c333b92",
+        "rev": "9d0eff868d328fe67c60c26c8ba50e0b9d8de867",
        "type": "github"
      },
      "original": {
@ -451,11 +414,11 @@
    },
    "nix-filter": {
      "locked": {
-        "lastModified": 1730207686,
-        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
        "owner": "numtide",
        "repo": "nix-filter",
-        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
        "type": "github"
      },
      "original": {
@ -464,51 +427,31 @@
        "type": "github"
      }
    },
-    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "attic",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
    "nix-super": {
      "inputs": {
        "flake-compat": "flake-compat_2",
-        "flake-parts": "flake-parts_3",
-        "git-hooks-nix": "git-hooks-nix",
+        "flake-parts": "flake-parts_2",
        "libgit2": "libgit2",
        "nixpkgs": "nixpkgs_3",
-        "nixpkgs-23-11": [
-          "blank"
-        ],
        "nixpkgs-regression": [
          "blank"
-        ]
+        ],
+        "pre-commit-hooks": "pre-commit-hooks"
      },
      "locked": {
-        "lastModified": 1731271232,
-        "narHash": "sha256-HSNUAfhtG/A2hnrzPMT3asZZ2Wb3dAmedOr1VzptOCg=",
-        "rev": "1eb19dd804a83d99c497118af8ab781eee569c65",
-        "type": "tarball",
-        "url": "https://forge.privatevoid.net/api/v1/repos/max/nix-super/archive/1eb19dd804a83d99c497118af8ab781eee569c65.tar.gz"
+        "host": "git.privatevoid.net",
+        "lastModified": 1713821351,
+        "narHash": "sha256-JctHGT1oa4pet4PgUKRM7pf0w+qGe0a/ahVij8bee3o=",
+        "owner": "max",
+        "repo": "nix-super",
+        "rev": "5ecd820c18b1aaa3c8ee257a7a9a2624c4107031",
+        "type": "gitlab"
      },
      "original": {
-        "type": "tarball",
-        "url": "https://forge.privatevoid.net/max/nix-super/archive/master.tar.gz"
+        "host": "git.privatevoid.net",
+        "owner": "max",
+        "repo": "nix-super",
+        "type": "gitlab"
      }
    },
    "nixpkgs": {
@ -545,27 +488,27 @@
    },
    "nixpkgs_3": {
      "locked": {
-        "lastModified": 1723688146,
-        "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=",
+        "lastModified": 1709083642,
+        "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0",
+        "rev": "b550fe4b4776908ac2a861124307045f8e717c8e",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "release-23.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs_4": {
      "locked": {
-        "lastModified": 1730785428,
-        "narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
+        "lastModified": 1722539632,
+        "narHash": "sha256-g4L+I8rDl7RQy5x8XcEMqNO49LFhrHTzVBqXtG2+FGo=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
+        "rev": "f2d6c7123138044e0c68902268bd8f37dd7e2fa7",
        "type": "github"
      },
      "original": {
@ -575,6 +518,38 @@
        "type": "github"
      }
    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "nix-super"
+        ],
+        "flake-utils": "flake-utils",
+        "gitignore": [
+          "nix-super"
+        ],
+        "nixpkgs": [
+          "nix-super",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "nix-super",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712897695,
+        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
    "repin-flake-utils": {
      "inputs": {
        "systems": [
@ -582,11 +557,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1726560853,
-        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
        "type": "github"
      },
      "original": {
@ -602,7 +577,7 @@
        "blank": "blank",
        "devshell": "devshell",
        "drv-parts": "drv-parts",
-        "flake-parts": "flake-parts_2",
+        "flake-parts": "flake-parts",
        "hercules-ci-agent": "hercules-ci-agent",
        "hercules-ci-effects": "hercules-ci-effects",
        "hyprspace": "hyprspace",
--- a/flake.nix
+++ b/flake.nix
@ -30,10 +30,9 @@
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";

    nix-super = {
-      url = "https://forge.privatevoid.net/max/nix-super/archive/master.tar.gz";
+      url = "gitlab:max/nix-super?host=git.privatevoid.net";
      inputs = {
        nixpkgs-regression.follows = "blank";
-        nixpkgs-23-11.follows = "blank";
      };
    };

@ -55,6 +54,7 @@
        nixpkgs.follows = "nixpkgs";
        nixpkgs-stable.follows = "nixpkgs";
        flake-compat.follows = "blank";
+        flake-utils.follows = "repin-flake-utils";
      };
    };

--- a/modules/consul-distributed-services/default.nix
+++ b/modules/consul-distributed-services/default.nix
@ -26,13 +26,11 @@ in

      cfg = v.distributed;

-      svcs = map (x: config.consul.services.${x}) cfg.registerServices;
+      svc = config.consul.services.${cfg.registerService};

      runWithRegistration = pkgs.writeShellScript "run-with-registration" ''
-        trap '${lib.concatStringsSep ";" (map (svc: svc.commands.deregister) svcs)}' EXIT
-        ${lib.concatStringsSep "\n" (
-          map (svc: svc.commands.register) svcs
-        )}
+        trap '${svc.commands.deregister}' EXIT
+        ${svc.commands.register}
        ''${@}
      '';

@ -51,10 +49,10 @@ in
      [Service]
      ExecStartPre=${waitForConsul} 'services/${n}%i'
      ExecStart=
-      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerServices != []) runWithRegistration} ${ExecStart}
+      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
      Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
      ${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
-      ${optionalString (cfg.registerServices != []) (lib.concatStringsSep "\n" (map (svc: "ExecStopPost=${svc.commands.deregister}") svcs))}
+      ${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
    ''))
  ];
 }
--- a/modules/motd/default.nix
+++ b/modules/motd/default.nix
@ -1,4 +1,4 @@
-{ cluster, config, depot, pkgs, ... }:
+{ config, depot, pkgs, ... }:
 {
  users.motd = builtins.readFile ./motd.txt;
  environment.interactiveShellInit = let
@ -8,11 +8,6 @@
    grep = exec pkgs.gnugrep "grep";
    countUsers = '' ${util "who"} -q | ${util "head"} -n1 | ${util "tr"} ' ' \\n | ${util "uniq"} | ${util "wc"} -l'';
    countSessions = '' ${util "who"} -q | ${util "head"} -n1 | ${util "wc"} -w'';
-
-    rev = if cluster.config.simulacrum then
-      "simulacrum"
-    else
-      depot.rev or "\${BRED}(✘)\${CO}\${BWHITE} Dirty";
  in ''
    (
    # Reset colors
@ -45,7 +40,7 @@
    echo -e " █ ''${BGREEN}(✓)''${CO} ''${BWHITE}You are using a genuine Private Void™ system.''${CO}"
    echo    " █"
    echo -e " █ ''${BWHITE}OS Version....:''${CO} NixOS ''${CAB}${config.system.nixos.version}''${CO}" 
-    echo -e " █ ''${BWHITE}Configuration.:''${CO} ''${CAB}${rev}''${CO}" 
+    echo -e " █ ''${BWHITE}Configuration.:''${CO} ''${CAB}${depot.rev or "\${BRED}(✘)\${CO}\${BWHITE} Dirty"}''${CO}" 
    echo -e " █ ''${BWHITE}Uptime........:''${CO} $(${uptime} -p | ${util "cut"} -d ' ' -f2- | GREP_COLORS='mt=01;35' ${grep} --color=always '[0-9]*')"
    echo -e " █ ''${BWHITE}SSH Logins....:''${CO} There are currently ''${CAB}$(${countUsers})''${CO} users logged in on ''${CAB}$(${countSessions})''${CO} sessions"
    )
--- a/modules/nix-config/server.nix
+++ b/modules/nix-config/server.nix
@ -14,7 +14,7 @@
      experimental-features = nix-command flakes cgroups
      use-cgroups = true
      builders-use-substitutes = true
-      flake-registry = https://registry.${depot.lib.meta.domain}/flake-registry.json
+      flake-registry = https://git.${depot.lib.meta.domain}/private-void/registry/-/raw/master/registry.json
      
      # For Hercules CI agent
      narinfo-cache-negative-ttl = 0
--- a/modules/part.nix
+++ b/modules/part.nix
@ -29,6 +29,7 @@ in
    reflection = ./reflection;
    shell-config = ./shell-config;
    ssh = ./ssh;
+    system-info = ./system-info;
    system-recovery = ./system-recovery;
    systemd-extras = ./systemd-extras;
    tested = ./tested;
@ -55,6 +56,7 @@ in
      motd
      networking
      nix-config-server
+      system-info
      system-recovery
      tested
    ];
--- a/modules/system-info/default.nix
+++ b/modules/system-info/default.nix
@ -0,0 +1,5 @@
+{ depot, ... }:
+
+{
+  system.configurationRevision = depot.rev or null;
+}
--- a/modules/systemd-extras/distributed.nix
+++ b/modules/systemd-extras/distributed.nix
@ -17,11 +17,6 @@ with lib;
          type = with types; nullOr str;
          default = null;
        };
-        registerServices = mkOption {
-          description = "Consul services to register when this service gets started.";
-          type = with types; listOf str;
-          default = if config.distributed.registerService == null then [ ] else [ config.distributed.registerService ];
-        };
      };
    }));
  };
--- a/packages/checks/default.nix
+++ b/packages/checks/default.nix
@ -7,6 +7,7 @@ let
 in

 {
+  debug = lib.warn "debug mode is enabled" true;
  perSystem = { filters, pkgs, self', system, ... }: {
    checks = lib.mkIf (system == "x86_64-linux") {
      ascensions = pkgs.callPackage ./ascensions.nix {
@ -29,6 +30,11 @@ in
        inherit (self'.packages) keycloak;
      };

+      patroni = pkgs.callPackage ./patroni.nix {
+        inherit (self) nixosModules;
+        inherit (self'.packages) postgresql;
+      };
+
      s3ql-upgrade = pkgs.callPackage ./s3ql-upgrade.nix {
        inherit (self'.packages) s3ql;
        inherit (self) nixosModules;
--- a/packages/checks/keycloak-custom-jre.nix
+++ b/packages/checks/keycloak-custom-jre.nix
@ -7,8 +7,7 @@ nixosTest {
    package = keycloak;
    database.passwordFile = builtins.toFile "keycloak-test-password" "kcnixostest1234";
    settings = {
-      http-enabled = true;
-      proxy-headers = "xforwarded";
+      proxy = "edge";
      hostname = "keycloak.local";
    };
  };
--- a/packages/checks/patroni.nix
+++ b/packages/checks/patroni.nix
@ -0,0 +1,211 @@
+{ nixosTest, nixosModules, postgresql }:
+
+# taken from https://github.com/phfroidmont/nixpkgs/blob/patroni-module/nixos/tests/patroni.nix
+nixosTest (
+  let
+    nodesIps = [
+      "192.168.1.1"
+      "192.168.1.2"
+      "192.168.1.3"
+    ];
+
+    createNode = index: { pkgs, ... }:
+      let
+        ip = builtins.elemAt nodesIps index; # since we already use IPs to identify servers
+      in
+      {
+        imports = [
+          nixosModules.patroni
+          nixosModules.systemd-extras
+        ];
+
+        networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
+          { address = ip; prefixLength = 16; }
+        ];
+
+        networking.firewall.allowedTCPPorts = [ 5432 8008 5010 ];
+
+        environment.systemPackages = [ pkgs.jq ];
+
+        services.patroni = {
+
+          enable = true;
+
+          postgresqlPackage = postgresql.withPackages (p: [ p.pg_safeupdate ]);
+
+          scope = "cluster1";
+          name = "node${toString(index + 1)}";
+          nodeIp = ip;
+          otherNodesIps = builtins.filter (h: h != ip) nodesIps;
+          softwareWatchdog = true;
+
+          settings = {
+            bootstrap = {
+              dcs = {
+                ttl = 30;
+                loop_wait = 10;
+                retry_timeout = 10;
+                maximum_lag_on_failover = 1048576;
+              };
+              initdb = [
+                { encoding = "UTF8"; }
+                "data-checksums"
+              ];
+            };
+
+            postgresql = {
+              use_pg_rewind = true;
+              use_slots = true;
+              authentication = {
+                replication = {
+                  username = "replicator";
+                };
+                superuser = {
+                  username = "postgres";
+                };
+                rewind = {
+                  username = "rewind";
+                };
+              };
+              parameters = {
+                listen_addresses = "${ip}";
+                wal_level = "replica";
+                hot_standby_feedback = "on";
+                unix_socket_directories = "/tmp";
+              };
+              pg_hba = [
+                "host replication replicator 192.168.1.0/24 md5"
+                # Unsafe, do not use for anything other than tests
+                "host all all 0.0.0.0/0 trust"
+              ];
+            };
+
+            etcd3 = {
+              host = "192.168.1.4:2379";
+            };
+          };
+
+          environmentFiles = {
+            PATRONI_REPLICATION_PASSWORD = pkgs.writeText "replication-password" "postgres";
+            PATRONI_SUPERUSER_PASSWORD = pkgs.writeText "superuser-password" "postgres";
+            PATRONI_REWIND_PASSWORD = pkgs.writeText "rewind-password" "postgres";
+          };
+        };
+
+        # We always want to restart so the tests never hang
+        systemd.services.patroni.serviceConfig.StartLimitIntervalSec = 0;
+      };
+  in
+  {
+    name = "patroni";
+
+    nodes = {
+      node1 = createNode 0;
+      node2 = createNode 1;
+      node3 = createNode 2;
+
+      etcd = { pkgs, ... }: {
+
+        networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
+          { address = "192.168.1.4"; prefixLength = 16; }
+        ];
+
+        services.etcd = {
+          enable = true;
+          listenClientUrls = [ "http://192.168.1.4:2379" ];
+        };
+
+        networking.firewall.allowedTCPPorts = [ 2379 ];
+      };
+
+      client = { pkgs, ... }: {
+        environment.systemPackages = [ postgresql ];
+
+        networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
+          { address = "192.168.2.1"; prefixLength = 16; }
+        ];
+
+        services.haproxy = {
+          enable = true;
+          config = ''
+            global
+                maxconn 100
+
+            defaults
+                log global
+                mode tcp
+                retries 2
+                timeout client 30m
+                timeout connect 4s
+                timeout server 30m
+                timeout check 5s
+
+            listen cluster1
+                bind 127.0.0.1:5432
+                option httpchk
+                http-check expect status 200
+                default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+                ${builtins.concatStringsSep "\n" (map (ip: "server postgresql_${ip}_5432 ${ip}:5432 maxconn 100 check port 8008") nodesIps)}
+          '';
+        };
+      };
+    };
+
+
+
+    testScript = ''
+      nodes = [node1, node2, node3]
+
+      def wait_for_all_nodes_ready(expected_replicas=2):
+          booted_nodes = filter(lambda node: node.booted, nodes)
+          for node in booted_nodes:
+              print(node.succeed("patronictl list cluster1"))
+              node.wait_until_succeeds(f"[ $(patronictl list -f json cluster1 | jq 'length') == {expected_replicas + 1} ]")
+              node.wait_until_succeeds("[ $(patronictl list -f json cluster1 | jq 'map(select(.Role | test(\"^Leader$\"))) | map(select(.State | test(\"^running$\"))) | length') == 1 ]")
+              node.wait_until_succeeds(f"[ $(patronictl list -f json cluster1 | jq 'map(select(.Role | test(\"^Replica$\"))) | map(select(.State | test(\"^streaming$\"))) | length') == {expected_replicas} ]")
+              print(node.succeed("patronictl list cluster1"))
+          client.wait_until_succeeds("psql -h 127.0.0.1 -U postgres --command='select 1;'")
+
+      def run_dummy_queries():
+          client.succeed("psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='insert into dummy(val) values (101);'")
+          client.succeed("test $(psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='select val from dummy where val = 101;') -eq 101")
+          client.succeed("psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='delete from dummy where val = 101;'")
+
+      start_all()
+
+      with subtest("should bootstrap a new patroni cluster"):
+          wait_for_all_nodes_ready()
+
+      with subtest("should be able to insert and select"):
+          client.succeed("psql -h 127.0.0.1 -U postgres --command='create table dummy as select * from generate_series(1, 100) as val;'")
+          client.succeed("test $(psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100")
+
+      with subtest("should restart after all nodes are crashed"):
+          for node in nodes:
+              node.crash()
+          for node in nodes:
+              node.start()
+          wait_for_all_nodes_ready()
+
+      with subtest("should be able to run queries while any one node is crashed"):
+          masterNodeName = node1.succeed("patronictl list -f json cluster1 | jq '.[] | select(.Role | test(\"^Leader$\")) | .Member' -r").strip()
+          masterNodeIndex = int(masterNodeName[len(masterNodeName)-1]) - 1
+
+          # Move master node at the end of the list to avoid multiple failovers (makes the test faster and more consistent)
+          nodes.append(nodes.pop(masterNodeIndex))
+
+          for node in nodes:
+              node.crash()
+              wait_for_all_nodes_ready(1)
+
+              # Execute some queries while a node is down.
+              run_dummy_queries()
+
+              # Restart crashed node.
+              node.start()
+              wait_for_all_nodes_ready()
+
+              # Execute some queries with the node back up.
+              run_dummy_queries()
+    '';
+  })
--- a/packages/checks/s3ql-upgrade.nix
+++ b/packages/checks/s3ql-upgrade.nix
@ -10,9 +10,6 @@ testers.runNixOSTest {
      nixosModules.systemd-extras
      ./modules/nixos/age-dummy-secrets
      ./modules/nixos/age-dummy-secrets/options.nix
-      {
-        options.services.locksmith = lib.mkSinkUndeclaredOptions { };
-      }
    ];

    _module.args.depot.packages = { inherit (previous.packages.${system}) s3ql; };
--- a/packages/patched-derivations.nix
+++ b/packages/patched-derivations.nix
@ -18,10 +18,6 @@ super: rec {
    };
  };

-  jitsi-meet-insecure = let
-    olm-insecure = assert builtins.length super.olm.meta.knownVulnerabilities > 0; super.olm.overrideAttrs (o: { meta = o.meta // { knownVulnerabilities = []; }; });
-  in super.jitsi-meet.override { olm = olm-insecure; };
-
  jre17_standard = let
    jre = super.jre_minimal.override {
      jdk = super.jdk17_headless;
@ -52,6 +48,7 @@ super: rec {

  s3ql = (patch super.s3ql "patches/base/s3ql").overrideAttrs (old: {
    propagatedBuildInputs = old.propagatedBuildInputs ++ [
+      super.python3Packages.packaging
      super.python3Packages.systemd
    ];
  });
--- a/packages/patched-inputs.nix
+++ b/packages/patched-inputs.nix
@ -11,6 +11,8 @@
      nix-super = packages.nix-super.nix;

      agenix = packages.agenix.agenix.override { nix = nix-super; };
+
+      hci = packages.hercules-ci-agent.hercules-ci-cli;
    };
  };
 }
--- a/packages/projects.nix
+++ b/packages/projects.nix
@ -59,6 +59,7 @@
        tools = with flakePkgs; [
          agenix
          graf
+          hci
          npins
          pin
          void
--- a/packages/shadows.nix
+++ b/packages/shadows.nix
@ -9,9 +9,7 @@

    options.shadows = lib.mkOption {
      type = with lib.types; lazyAttrsOf package;
-      default = {
-        jitsi-meet = self'.packages.jitsi-meet-insecure;
-      };
+      default = { };
    };
  };
 }
--- a/packages/system-filter.nix
+++ b/packages/system-filter.nix
@ -1,5 +1,6 @@
 {
  packages = {
+    hci = [ "x86_64-linux" ];
    hydra = [ "x86_64-linux" ];
    jellyfin = [ "x86_64-linux" ];
    keycloak = [ "x86_64-linux" ];
--- a/patches/base/kanidm/unixd-authenticated.patch
+++ b/patches/base/kanidm/unixd-authenticated.patch
@ -1,28 +1,25 @@
-diff --git a/unix_integration/resolver/src/idprovider/kanidm.rs b/unix_integration/resolver/src/idprovider/kanidm.rs
-index 63cedb4d5..35c45fb0e 100644
--- a/unix_integration/resolver/src/idprovider/kanidm.rs
-+++ b/unix_integration/resolver/src/idprovider/kanidm.rs
-@@ -7,6 +7,7 @@ use kanidm_proto::internal::OperationError;
+diff --git a/unix_integration/src/idprovider/kanidm.rs b/unix_integration/src/idprovider/kanidm.rs
+index 6fc015756..31593f03e 100644
+--- a/unix_integration/src/idprovider/kanidm.rs
+++ b/unix_integration/src/idprovider/kanidm.rs
+@@ -4,6 +4,7 @@ use kanidm_client::{ClientError, KanidmClient, StatusCode};
+ use kanidm_proto::internal::OperationError;
 use kanidm_proto::v1::{UnixGroupToken, UnixUserToken};
- use std::collections::BTreeSet;
- use std::time::{Duration, SystemTime};
+ use tokio::sync::{broadcast, RwLock};
 +use std::env;
- use tokio::sync::{broadcast, Mutex};
 
- use kanidm_lib_crypto::CryptoPolicy;
-@@ -38,6 +39,8 @@ struct KanidmProviderInternal {
-     hmac_key: HmacKey,
-     crypto_policy: CryptoPolicy,
-     pam_allow_groups: BTreeSet<String>,
+ use super::interface::{
+     // KeyStore,
+@@ -25,12 +26,28 @@ const TAG_IDKEY: &str = "idkey";
+ 
+ pub struct KanidmProvider {
+     client: RwLock<KanidmClient>,
 +    auth_name: Option<String>,
 +    auth_password: Option<String>,
 }
 
- pub struct KanidmProvider {
-@@ -102,6 +105,19 @@ impl KanidmProvider {
-             .map(|GroupMap { local, with }| (local, Id::Name(with)))
-             .collect();
- 
+ impl KanidmProvider {
+     pub fn new(client: KanidmClient) -> Self {
 +        let env_username: Option<String>;
 +        let env_password: Option<String>;
 +        match (env::var_os("KANIDM_NAME"), env::var_os("KANIDM_PASSWORD")) {
@ -35,29 +32,23 @@ index 63cedb4d5..35c45fb0e 100644
 +                env_password = None;
 +            }
 +        }
-+
-         Ok(KanidmProvider {
-             inner: Mutex::new(KanidmProviderInternal {
-                 state: CacheState::OfflineNextCheck(now),
-@@ -109,6 +125,8 @@ impl KanidmProvider {
-                 hmac_key,
-                 crypto_policy,
-                 pam_allow_groups,
-+                auth_name: env_username,
-+                auth_password: env_password
-             }),
-             map_group,
-         })
-@@ -256,7 +274,11 @@ impl KanidmProviderInternal {
+         KanidmProvider {
+             client: RwLock::new(client),
+            auth_name: env_username,
+            auth_password: env_password,
+         }
     }
+ }
+@@ -118,7 +135,11 @@ impl IdProvider for KanidmProvider {
 
-     async fn attempt_online(&mut self, _tpm: &mut tpm::BoxedDynTpm, now: SystemTime) -> bool {
-        match self.client.auth_anonymous().await {
+     // Needs .read on all types except re-auth.
+     async fn provider_authenticate(&self, _tpm: &mut tpm::BoxedDynTpm) -> Result<(), IdpError> {
+-        match self.client.write().await.auth_anonymous().await {
 +        let auth_method = match (&self.auth_name, &self.auth_password) {
-+            (Some(name), Some(password)) => self.client.auth_simple_password(name, password).await,
-+            _ => self.client.auth_anonymous().await
+            (Some(name), Some(password)) => self.client.write().await.auth_simple_password(name, password).await,
+            _ => self.client.write().await.auth_anonymous().await
 +        };
 +        match auth_method {
-             Ok(_uat) => {
-                 self.state = CacheState::Online;
-                 true
+             Ok(_uat) => Ok(()),
+             Err(err) => {
+                 error!(?err, "Provider authentication failed");
--- a/patches/base/s3ql/0000-cache-entry-seek-whence.patch
+++ b/patches/base/s3ql/0000-cache-entry-seek-whence.patch
@ -0,0 +1,18 @@
+diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py
+index a4b55fd1..267b9a12 100644
+--- a/src/s3ql/block_cache.py
+++ b/src/s3ql/block_cache.py
+@@ -86,10 +86,10 @@ class CacheEntry:
+     def flush(self):
+         self.fh.flush()
+ 
+-    def seek(self, off):
+    def seek(self, off, whence=0):
+         if self.pos != off:
+-            self.fh.seek(off)
+-            self.pos = off
+            self.fh.seek(off, whence)
+            self.pos = self.fh.tell()
+ 
+     def tell(self):
+         return self.pos
--- a/patches/base/s3ql/0001-fix-plain-block-size.patch
+++ b/patches/base/s3ql/0001-fix-plain-block-size.patch
@ -0,0 +1,26 @@
+diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
+index 6402fec1..9ed3627e 100644
+--- a/src/s3ql/backends/comprenc.py
+++ b/src/s3ql/backends/comprenc.py
+@@ -276,7 +276,7 @@ class ComprencBackend(AbstractBackend):
+             buf.seek(0)
+             fh = buf
+ 
+-        return self.backend.write_fh(key, fh, meta_raw)
+        return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
+ 
+     def contains(self, key):
+         return self.backend.contains(key)
+diff --git a/src/s3ql/database.py b/src/s3ql/database.py
+index bb4054e6..c2142bf6 100644
+--- a/src/s3ql/database.py
+++ b/src/s3ql/database.py
+@@ -659,7 +659,7 @@ def upload_metadata(
+             )
+             obj = METADATA_OBJ_NAME % (blockno, params.seq_no)
+             fh.seek(blockno * blocksize)
+-            backend.write_fh(obj, fh, len_=blocksize)
+            backend.write_fh(obj, fh, len_=min(blocksize, db_size - blockno * blocksize))
+ 
+         if not update_params:
+             return
--- a/patches/base/s3ql/0002-comprenc-always-copy.patch
+++ b/patches/base/s3ql/0002-comprenc-always-copy.patch
@ -0,0 +1,17 @@
+diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
+index 9ed3627e..db419bb7 100644
+--- a/src/s3ql/backends/comprenc.py
+++ b/src/s3ql/backends/comprenc.py
+@@ -276,6 +276,12 @@ class ComprencBackend(AbstractBackend):
+             buf.seek(0)
+             fh = buf
+ 
+        if meta_raw['compression'] == 'None' and meta_raw['encryption'] == 'None':
+            buf = io.BytesIO()
+            copyfh(fh, buf, len_)
+            buf.seek(0)
+            fh = buf
+        
+         return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
+ 
+     def contains(self, key):
--- a/patches/base/s3ql/metadata-accurate-length.patch
+++ b/patches/base/s3ql/metadata-accurate-length.patch
@ -1,13 +0,0 @@
-diff --git a/src/s3ql/database.py b/src/s3ql/database.py
-index 1c6df119..f3a47781 100644
--- a/src/s3ql/database.py
-+++ b/src/s3ql/database.py
-@@ -677,7 +677,7 @@ def upload_metadata(
-             )
-             obj = METADATA_OBJ_NAME % (blockno, params.seq_no)
-             fh.seek(blockno * blocksize)
-            backend.write_fh(obj, fh, len_=blocksize)
-+            backend.write_fh(obj, fh, len_=min(blocksize, db_size - blockno * blocksize))
- 
-         if not update_params:
-             return
--- a/patches/base/s3ql/remove-ssl-monkeypatch.patch
+++ b/patches/base/s3ql/remove-ssl-monkeypatch.patch
@ -0,0 +1,12 @@
+diff --git a/tests/t0_http.py b/tests/t0_http.py
+index 66ed564f..36bebab1 100755
+--- a/tests/t0_http.py
+++ b/tests/t0_http.py
+@@ -289,7 +289,6 @@ def do_GET(self):
+ 
+     # We don't *actually* want to establish SSL, that'd be
+     # to complex for our mock server
+-    monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
+     conn = HTTPConnection(
+         test_host,
+         test_port,
--- a/patches/base/s3ql/s3c-accurate-length.patch
+++ b/patches/base/s3ql/s3c-accurate-length.patch
@ -0,0 +1,26 @@
+commit 1edbbcf08d5701ea38f13fca7491418318aebca9
+Author: Max <max@privatevoid.net>
+Date:   Fri Jun 7 23:31:08 2024 +0200
+
+    accurate length
+
+diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py
+index 2995ca4f..3c3c79ab 100644
+--- a/src/s3ql/backends/s3c.py
+++ b/src/s3ql/backends/s3c.py
+@@ -387,9 +387,13 @@ class Backend(AbstractBackend):
+         '''
+ 
+         off = fh.tell()
+        fh.seek(0, os.SEEK_END)
+        actual_len = fh.tell() - off
+        fh.seek(off, os.SEEK_SET)
+         if len_ is None:
+-            fh.seek(0, os.SEEK_END)
+-            len_ = fh.tell()
+            len_ = actual_len
+        else:
+            len_ = min(len_, actual_len)
+         return self._write_fh(key, fh, off, len_, metadata or {})
+ 
+     @retry
--- a/patches/base/s3ql/s3v4.patch
+++ b/patches/base/s3ql/s3v4.patch
@ -0,0 +1,392 @@
+From 11e3a9cea77cd8498d874f7fd69a938af4da68cd Mon Sep 17 00:00:00 2001
+From: xeji <36407913+xeji@users.noreply.github.com>
+Date: Thu, 28 Mar 2024 22:19:11 +0100
+Subject: [PATCH] new backend s3c4: s3c with V4 request signatures (#349)
+
+---
+ rst/backends.rst              |  15 ++++
+ src/s3ql/backends/__init__.py |   3 +-
+ src/s3ql/backends/s3.py       | 100 ++----------------------
+ src/s3ql/backends/s3c4.py     | 140 ++++++++++++++++++++++++++++++++++
+ src/s3ql/parse_args.py        |   2 +-
+ tests/mock_server.py          |  11 +++
+ 6 files changed, 174 insertions(+), 97 deletions(-)
+ create mode 100644 src/s3ql/backends/s3c4.py
+
+diff --git a/rst/backends.rst b/rst/backends.rst
+index 7220ee96..4bc68387 100644
+--- a/rst/backends.rst
+++ b/rst/backends.rst
+@@ -341,6 +341,14 @@ can be an arbitrary prefix that will be prepended to all object names
+ used by S3QL. This allows you to store several S3QL file systems in
+ the same bucket.
+ 
+`s3c://` authenticates API requests using AWS V2 signatures, which are
+deprecated by AWS but still accepted by many S3 compatible services.
+
+`s3c4://` denotes a variant of this backend that works the same
+but uses AWS V4 signatures for request authentication instead: ::
+
+   s3c4://<hostname>:<port>/<bucketname>/<prefix>
+
+ The S3 compatible backend accepts the following backend options:
+ 
+ .. option:: no-ssl
+@@ -385,6 +393,13 @@ The S3 compatible backend accepts the following backend options:
+    necessary if your storage server does not return a valid response
+    body for a successful copy operation.
+ 
+.. option:: sig-region=<region>
+
+   For `s3c4://` variant only: Region to use for calculating V4
+   request signatures. Contrary to S3, the region is not a defined 
+   part of the storage URL and must be specified separately.
+   Defaults to `us-east-1`.
+
+ .. _`S3 COPY API`: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
+ .. __: https://doc.s3.amazonaws.com/proposals/copy.html
+ 
+diff --git a/src/s3ql/backends/__init__.py b/src/s3ql/backends/__init__.py
+index a1335762..442828cd 100644
+--- a/src/s3ql/backends/__init__.py
+++ b/src/s3ql/backends/__init__.py
+@@ -6,7 +6,7 @@
+ This work can be distributed under the terms of the GNU GPLv3.
+ '''
+ 
+-from . import gs, local, rackspace, s3, s3c, swift, swiftks
+from . import gs, local, rackspace, s3, s3c, s3c4, swift, swiftks
+ from .b2.b2_backend import B2Backend
+ 
+ #: Mapping from storage URL prefixes to backend classes
+@@ -15,6 +15,7 @@
+     'local': local.Backend,
+     'gs': gs.Backend,
+     's3c': s3c.Backend,
+    's3c4': s3c4.Backend,
+     'swift': swift.Backend,
+     'swiftks': swiftks.Backend,
+     'rackspace': rackspace.Backend,
+diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py
+index e05a49ba..5548a855 100644
+--- a/src/s3ql/backends/s3.py
+++ b/src/s3ql/backends/s3.py
+@@ -15,7 +15,7 @@
+ from xml.sax.saxutils import escape as xml_escape
+ 
+ from ..logging import QuietError
+-from . import s3c
+from . import s3c4
+ from .common import retry
+ from .s3c import get_S3Error
+ 
+@@ -28,22 +28,23 @@
+ # pylint: disable=E1002,E1101
+ 
+ 
+-class Backend(s3c.Backend):
+class Backend(s3c4.Backend):
+     """A backend to store data in Amazon S3
+ 
+     This class uses standard HTTP connections to connect to S3.
+     """
+ 
+-    known_options = (s3c.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
+    known_options = (s3c4.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
+         'dumb-copy',
+         'disable-expect100',
+        'sig-region',
+     }
+ 
+     def __init__(self, options):
+         self.region = None
+-        self.signing_key = None
+         super().__init__(options)
+         self._set_storage_options(self._extra_put_headers)
+        self.sig_region = self.region
+ 
+     def _parse_storage_url(self, storage_url, ssl_context):
+         hit = re.match(r'^s3s?://([^/]+)/([^/]+)(?:/(.*))?$', storage_url)
+@@ -147,94 +148,3 @@ def _delete_multi(self, keys):
+ 
+         except:
+             self.conn.discard()
+-
+-    def _authorize_request(self, method, path, headers, subres, query_string):
+-        '''Add authorization information to *headers*'''
+-
+-        # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+-
+-        now = time.gmtime()
+-        # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
+-        #                    '%a, %d %b %Y %H:%M:%S GMT')
+-
+-        ymd = time.strftime('%Y%m%d', now)
+-        ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
+-
+-        headers['x-amz-date'] = ymdhms
+-        headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
+-        # headers['x-amz-content-sha256'] = hashlib.sha256(body).hexdigest()
+-        headers.pop('Authorization', None)
+-
+-        auth_strs = [method]
+-        auth_strs.append(urllib.parse.quote(path))
+-
+-        if query_string:
+-            s = urllib.parse.urlencode(
+-                query_string, doseq=True, quote_via=urllib.parse.quote
+-            ).split('&')
+-        else:
+-            s = []
+-        if subres:
+-            s.append(urllib.parse.quote(subres) + '=')
+-        if s:
+-            s = '&'.join(sorted(s))
+-        else:
+-            s = ''
+-        auth_strs.append(s)
+-
+-        # Headers
+-        sig_hdrs = sorted(x.lower() for x in headers.keys())
+-        for hdr in sig_hdrs:
+-            auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
+-        auth_strs.append('')
+-        auth_strs.append(';'.join(sig_hdrs))
+-        auth_strs.append(headers['x-amz-content-sha256'])
+-        can_req = '\n'.join(auth_strs)
+-        # log.debug('canonical request: %s', can_req)
+-
+-        can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
+-        str_to_sign = (
+-            "AWS4-HMAC-SHA256\n"
+-            + ymdhms
+-            + '\n'
+-            + '%s/%s/s3/aws4_request\n' % (ymd, self.region)
+-            + can_req_hash
+-        )
+-        # log.debug('string to sign: %s', str_to_sign)
+-
+-        if self.signing_key is None or self.signing_key[1] != ymd:
+-            self.update_signing_key(ymd)
+-        signing_key = self.signing_key[0]
+-
+-        sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
+-
+-        cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
+-            self.login,
+-            now.tm_year,
+-            now.tm_mon,
+-            now.tm_mday,
+-            self.region,
+-        )
+-
+-        headers['Authorization'] = (
+-            'AWS4-HMAC-SHA256 '
+-            'Credential=%s,'
+-            'SignedHeaders=%s,'
+-            'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
+-        )
+-
+-    def update_signing_key(self, ymd):
+-        date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
+-        region_key = hmac_sha256(date_key, self.region.encode())
+-        service_key = hmac_sha256(region_key, b's3')
+-        signing_key = hmac_sha256(service_key, b'aws4_request')
+-
+-        self.signing_key = (signing_key, ymd)
+-
+-
+-def hmac_sha256(key, msg, hex=False):
+-    d = hmac.new(key, msg, hashlib.sha256)
+-    if hex:
+-        return d.hexdigest()
+-    else:
+-        return d.digest()
+diff --git a/src/s3ql/backends/s3c4.py b/src/s3ql/backends/s3c4.py
+new file mode 100644
+index 00000000..37ff0b7a
+--- /dev/null
+++ b/src/s3ql/backends/s3c4.py
+@@ -0,0 +1,140 @@
+'''
+s3c4.py - this file is part of S3QL.
+
+Copyright © 2008 Nikolaus Rath <Nikolaus@rath.org>
+
+This work can be distributed under the terms of the GNU GPLv3.
+'''
+
+import hashlib
+import hmac
+import logging
+import re
+import time
+import urllib.parse
+from xml.sax.saxutils import escape as xml_escape
+
+from ..logging import QuietError
+from . import s3c
+from .common import retry
+from .s3c import get_S3Error
+
+log = logging.getLogger(__name__)
+
+# Maximum number of keys that can be deleted at once
+MAX_KEYS = 1000
+
+# Pylint goes berserk with false positives
+# pylint: disable=E1002,E1101
+
+
+class Backend(s3c.Backend):
+    """A backend to stored data in some S3 compatible storage service.
+
+    This classes uses AWS Signature V4 for authorization.
+    """
+
+    known_options = s3c.Backend.known_options | {'sig-region'}
+
+    def __init__(self, options):
+        self.sig_region = options.backend_options.get('sig-region', 'us-east-1')
+        self.signing_key = None
+        super().__init__(options)
+
+    def __str__(self):
+        return 's3c4://%s/%s/%s' % (self.hostname, self.bucket_name, self.prefix)
+
+    def _authorize_request(self, method, path, headers, subres, query_string):
+        '''Add authorization information to *headers*'''
+
+        # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+
+        now = time.gmtime()
+        # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
+        #                    '%a, %d %b %Y %H:%M:%S GMT')
+
+        ymd = time.strftime('%Y%m%d', now)
+        ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
+
+        # add non-standard port to host header, needed for correct signature
+        if self.port != 443:
+            headers['host'] = '%s:%s' % (self.hostname, self.port)
+
+        headers['x-amz-date'] = ymdhms
+        headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
+
+        headers.pop('Authorization', None)
+
+        auth_strs = [method]
+        auth_strs.append(urllib.parse.quote(path))
+
+        if query_string:
+            s = urllib.parse.urlencode(
+                query_string, doseq=True, quote_via=urllib.parse.quote
+            ).split('&')
+        else:
+            s = []
+        if subres:
+            s.append(urllib.parse.quote(subres) + '=')
+        if s:
+            s = '&'.join(sorted(s))
+        else:
+            s = ''
+        auth_strs.append(s)
+
+        # Headers
+        sig_hdrs = sorted(x.lower() for x in headers.keys())
+        for hdr in sig_hdrs:
+            auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
+        auth_strs.append('')
+        auth_strs.append(';'.join(sig_hdrs))
+        auth_strs.append(headers['x-amz-content-sha256'])
+        can_req = '\n'.join(auth_strs)
+        # log.debug('canonical request: %s', can_req)
+
+        can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
+        str_to_sign = (
+            "AWS4-HMAC-SHA256\n"
+            + ymdhms
+            + '\n'
+            + '%s/%s/s3/aws4_request\n' % (ymd, self.sig_region)
+            + can_req_hash
+        )
+        # log.debug('string to sign: %s', str_to_sign)
+
+        if self.signing_key is None or self.signing_key[1] != ymd:
+            self.update_signing_key(ymd)
+        signing_key = self.signing_key[0]
+
+        sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
+
+        cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
+            self.login,
+            now.tm_year,
+            now.tm_mon,
+            now.tm_mday,
+            self.sig_region,
+        )
+
+        headers['Authorization'] = (
+            'AWS4-HMAC-SHA256 '
+            'Credential=%s,'
+            'SignedHeaders=%s,'
+            'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
+        )
+
+    def update_signing_key(self, ymd):
+        date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
+        region_key = hmac_sha256(date_key, self.sig_region.encode())
+        service_key = hmac_sha256(region_key, b's3')
+        signing_key = hmac_sha256(service_key, b'aws4_request')
+
+        self.signing_key = (signing_key, ymd)
+
+
+def hmac_sha256(key, msg, hex=False):
+    d = hmac.new(key, msg, hashlib.sha256)
+    if hex:
+        return d.hexdigest()
+    else:
+        return d.digest()
+diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py
+index 272e10c7..24ad50f4 100644
+--- a/src/s3ql/parse_args.py
+++ b/src/s3ql/parse_args.py
+@@ -374,7 +374,7 @@ def storage_url_type(s):
+     # slash (even when using a prefix), but we can't do that now because it
+     # would make file systems created without trailing slash inaccessible.
+     if re.match(r'^(s3|gs)://[^/]+$', s) or re.match(
+-        r'^(s3c|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
+        r'^(s3c|s3c4|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
+     ):
+         s += '/'
+ 
+diff --git a/tests/mock_server.py b/tests/mock_server.py
+index b453e705..e3084065 100644
+--- a/tests/mock_server.py
+++ b/tests/mock_server.py
+@@ -292,6 +292,16 @@ def send_error(self, status, message=None, code='', resource='', extra_headers=N
+             self.wfile.write(content)
+ 
+ 
+class S3C4RequestHandler(S3CRequestHandler):
+    '''Request Handler for s3c4 backend
+
+    Currently identical to S3CRequestHandler since mock request handlers
+    do not check request signatures.
+    '''
+
+    pass
+
+
+ class BasicSwiftRequestHandler(S3CRequestHandler):
+     '''A request handler implementing a subset of the OpenStack Swift Interface
+ 
+@@ -569,6 +579,7 @@ def inline_error(http_status, body):
+ #: corresponding storage urls
+ handler_list = [
+     (S3CRequestHandler, 's3c://%(host)s:%(port)d/s3ql_test'),
+    (S3C4RequestHandler, 's3c4://%(host)s:%(port)d/s3ql_test'),
+     # Special syntax only for testing against mock server
+     (BasicSwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),
+     (CopySwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),
--- a/secrets.nix
+++ b/secrets.nix
@ -16,6 +16,13 @@ in with hosts;
  "cluster/services/storage/secrets/garage-rpc-secret.age".publicKeys = max ++ map systemKeys [ grail VEGAS prophet ];
  "cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ grail VEGAS prophet ];
  "secrets/dovecot-ldap-token.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "secrets/gitlab-db-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "secrets/gitlab-initial-root-password.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "secrets/gitlab-openid-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "secrets/gitlab-secret-db.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "secrets/gitlab-secret-jws.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "secrets/gitlab-secret-otp.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "secrets/gitlab-secret-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ];
  "secrets/hyprspace-key-checkmate.age".publicKeys = max ++ map systemKeys [ checkmate ];
  "secrets/hyprspace-key-grail.age".publicKeys = max ++ map systemKeys [ grail ];
  "secrets/hyprspace-key-thunderskin.age".publicKeys = max ++ map systemKeys [ thunderskin ];
--- a/secrets/gitlab-db-credentials.age
+++ b/secrets/gitlab-db-credentials.age
--- a/secrets/gitlab-initial-root-password.age
+++ b/secrets/gitlab-initial-root-password.age
@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A XRrOSniXZk7nvufR3liJ3ocjX257eenhQUYZdlYCpl4
+ctZGdEgc9SgWka/3R/2WW4G9m1DHIk7HLKaBNyUeHtE
+-> ssh-ed25519 5/zT0w k3z9vLsjCPABV2kTRMC3xiriW+4BwSdvnk02Xtoi3zk
+w43L1pm8VvwxVp6k8NJA73afZtPGfD8eCb2koa2goZQ
+-> ssh-ed25519 d3WGuA Bi1l2WS3kL5Y5NoVh7jAja3BG9LXxem801SSR76j52s
+fKhRIb+Ug3sW4JI2rczNnh3Frx/EEnbQfhTUGdwLSo8
+-> AOy-grease dju$ xL|5Hh q(A
+h0bIKBg8yQBMqNR8M9DlA/wZWWFB+sdo4ApLXvTT19Moz3E5Vly8N2XKHrV3ggCE
+Vn2a3snrXDrWxqQgfQEfJo7FnydItRcgO7ZDOuNAlnooyk0
+--- 9bMYjHMQsJt4fqnmE2ezRzN4AoKIrlRKAqh8pYRw8SQ
+øÜ™‹j‡>ü‘râ|ˆ>˜º<CB9C>–QÌ7¬p²¾ïÐdð¤hëÝÏ Î3œü»€¤ÃÐÿ57´âð˜{ïžZ9áLš´ééÖ$DU$—0YÙ º3ÐBMÍã‰ü@oáªU¶_ßÁ¡dÅDÝ¶<C39D>5jq/¿‰…j’`›6›<36>Z‡îi—åAÄÞ&Q¯”œ¬¢Ê¡*Õ•:R%+ ôò<C3B4>É¡ù£Ì
--- a/secrets/gitlab-openid-secret.age
+++ b/secrets/gitlab-openid-secret.age
@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A ZCflrN3Tm5CiGr6ajyHWUBB/tQqvBuZkwTrJDrd/aV0
+ItnkxqiZTCT77SDnG0JgzaQlDL3LZ96V+kzjxjAJx5s
+-> ssh-ed25519 5/zT0w WoKnbgmzpR+HuLdXYCOkPfScle7g7U+NGA/YAmyfIhk
+pNfp+gOVyTfnXpVDRXuk16RyjlWjDILrO7Gibh7nRmU
+-> ssh-ed25519 d3WGuA L5xjtPNva83jZWsu2bCbcgaDNlou5BFVMsFkR8+L+2Q
+4+UtIsyOgY0NAuHtdg4lBJwMyZWquRsmRNeQ+YXqeA0
+-> hD-grease q%QV%; &/
+jl4ZKGU+SBSR0xhJN0yz7sV2uW/+Yhw
+--- 1LIvBjAzD1lUotPXuI4cPHSfUsMFbEaGjE/t+KnQcW4
+AWeûÛ¨Ë¯e¤	c[ ÖÌ3mÁíyÍÎˆÐñè6½
g{7›rd€_Ê7ØWPö©':ð¢uË›ùá¨N
--- a/secrets/gitlab-secret-db.age
+++ b/secrets/gitlab-secret-db.age
--- a/secrets/gitlab-secret-jws.age
+++ b/secrets/gitlab-secret-jws.age
--- a/secrets/gitlab-secret-otp.age
+++ b/secrets/gitlab-secret-otp.age
@ -0,0 +1,14 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A su6ATd6CDJ/TD/nAPw1K4ZmELBDdNLZI63DsZl0zCF0
+J+2ZXXZArtjDDLIaQL6HaEdawHo8tonMdzHf45IQMO4
+-> ssh-ed25519 5/zT0w wdKMnoA5/huvtT/jyj1Aixf9nKtkzcyPSs1yoUpxoAk
+yGiW4Zg0h4NGkdU0BZiWzC+72CJZK6pJdrSBuZCVGAE
+-> ssh-ed25519 d3WGuA p4QVeohmXdTo8v0Wh2pkEoyqMhZhmdrblBpq39ENnVk
+7TybdsMNokMu+2q5ESnvdcNwAeWTl/5XGZltzJ7etjI
+-> Q-grease KJL\,Pw& c!aOPX
+C6DVdLd90RXPgjf22U5Y8OsW9O9rkfE3kY0LGQhmmjCSZ7yHde4bhOAVNeNronxE
+xFy8GtD+ZllI4NPUSyl3Y/90//H2fVUb32WA3Ga5WJmksrGXzg
+--- yWDk0jbHXLxwE9jWTT85ORZy0Pw20jaRVihmkKfGnKo
+@# 
+Q)F:ÀŽ¤¶GÍû	#ógÒº¡¤«L…Ê-k{Tëd+˜´8žàÜƒüäá/è¹-Žaæ…Ë\O*—°!^Réãy÷›@Z/o™~I€
+œ[ô°¼PO’Â'vüše^ø,…?¢»Òo¼¸MÆ]1WƒËFò‹JëÄ™Ññ¨ôBý&y¼
yŸìVv‘_<E28098> %‹ûÇ<C3BB>«'
--- a/secrets/gitlab-secret-secret.age
+++ b/secrets/gitlab-secret-secret.age
--- a/secrets/keycloak-dbpass.age
+++ b/secrets/keycloak-dbpass.age
Author	SHA1	Message	Date
Max	00f233e8a5	cluster/services/frangiclave: funny	2024-08-12 03:04:14 +02:00
Max	e2fe73039c	cluster/services/frangiclave: test in simulacrum WIP	2024-08-12 03:04:14 +02:00
Max	e4f09218d5	cluster/services/frangiclave: retry_join	2024-08-12 03:04:14 +02:00
Max	ebc9b88c8d	cluster/services/frangiclave: some cluster stuff	2024-08-12 03:04:14 +02:00
Max	ea3414c427	cluster/services/frangiclave: init trivial WIP	2024-08-12 03:04:14 +02:00
Max	6eee030b7a	cluster/services/storage: register existing keys and buckets in incandescence	2024-08-12 03:04:03 +02:00
Max	75cebf4ed6	cluster/services/incandescence: add base layout for ascensions	2024-08-12 03:04:03 +02:00
Max	bc3cd82731	cluster/services/consul: implement runConsul incantation	2024-08-12 03:04:03 +02:00
Max	9cdf964c6c	cluster/services/forge: define db	2024-08-12 03:04:03 +02:00
Max	bb22fe0176	cluster/services/patroni: implement raw format for locksmith provider	2024-08-12 03:04:03 +02:00
Max	d1f2bc1227	cluster/services/storage: define snakeoil passphrase for heresy, ensure encryption	2024-08-12 03:04:03 +02:00
Max	a48ee00f3d	cluster/services/ways: add simulacrum deps	2024-08-12 03:04:03 +02:00
Max	9ed3655ccf	cluster/services/storage: use recursive simulacrum deps	2024-08-12 03:04:03 +02:00
Max	945698a3ea	cluster/services/patroni: add simulacrum deps	2024-08-12 03:04:03 +02:00
Max	f75c7b8522	cluster/services/locksmith: add simulacrum deps	2024-08-12 03:04:03 +02:00
Max	b258bab23e	cluster/services/incandescence: add simulacrum deps	2024-08-12 03:04:03 +02:00
Max	e2296eace7	cluster/services/chant: add simulacrum deps	2024-08-12 03:04:03 +02:00
Max	304ae6e53c	cluster/simulacrum: recursive service deps	2024-08-12 03:04:03 +02:00
Max	f322208f66	cluster/services/acme-client: implement augment for external ACME services	2024-08-12 03:04:03 +02:00
Max	7c4615ecfb	cluster/simulacrum: implement nowhere, fix networking	2024-08-12 03:04:03 +02:00
Max	ec38e10fa9	cluster/services/forge: use forService	2024-08-12 03:04:03 +02:00
Max	5d9ff62afe	cluster/services/dns: use patroni incandescence	2024-08-12 03:04:03 +02:00
Max	6d78b69601	cluster/services/patroni: implement incandescence provider for databases and users	2024-08-12 03:04:03 +02:00
Max	7129d44078	cluster/services/locksmith: only run secret generation command once	2024-08-12 03:04:03 +02:00
Max	76d205d114	cluster/services/locksmith: support skipping secret updates	2024-08-12 03:04:03 +02:00
Max	c8c9a6fbce	modules/external-storage: implement detectFs for s3c4	2024-08-12 03:04:03 +02:00
Max	a1cad2efcd	cluster/services/storage: use locksmith secrets for external storage	2024-08-12 03:04:03 +02:00
Max	c7f4e59908	cluster/services/storage: adjust test	2024-08-12 03:04:03 +02:00
Max	baed1ce871	cluster/services/storage: use incandescence	2024-08-12 03:04:03 +02:00
Max	014c1f9cd2	cluster/services/incandescence: init	2024-08-12 03:04:03 +02:00
Max	34704c8f08	modules/external-storage: support locksmith secrets	2024-08-12 03:04:03 +02:00
Max	ccc2a47880	cluster/services/storage: implement s3ql key format	2024-08-12 03:04:03 +02:00
Max	05cd729e90	cluster/services/hercules-ci-multi-agent: use forService	2024-08-12 03:04:03 +02:00
Max	8d0a2f00cc	cluster/services/monitoring: use forService	2024-08-12 03:04:03 +02:00
Max	ff26e1ebc1	checks/garage: drop	2024-08-12 03:04:03 +02:00
Max	b848084dd8	packages/catalog: expose simulacrum checks differently	2024-08-12 03:04:03 +02:00
Max	fe8ddd4094	cluster/simulacrum: expose checks	2024-08-12 03:04:03 +02:00
Max	030b680b33	cluster/services/forge: use forService	2024-08-12 03:04:03 +02:00
Max	b453b0bb21	cluster/services/attic: use forService	2024-08-12 03:04:03 +02:00
Max	b6e0390555	cluster/lib: implement config.lib.forService for better option filtering	2024-08-12 03:04:03 +02:00
Max	bbe3373c2e	cluster/simulacrum: set testConfig	2024-08-12 03:04:03 +02:00
Max	0ed4870b65	cluster/lib: introduce testConfig	2024-08-12 03:04:03 +02:00
Max	8ec13f5c87	cluster/services/storage: test in simulacrum	2024-08-12 03:04:03 +02:00
Max	5d52f72940	cluster/services/consul: test in simulacrum	2024-08-12 03:04:03 +02:00
Max	1af67b80ed	cluster/services/wireguard: make simulacrum compatible	2024-08-12 03:04:03 +02:00
Max	a810717843	cluster/catalog: support snakeoil secrets	2024-08-12 03:04:03 +02:00
Max	bd39fc5d07	cluster/simulacrum: init	2024-08-12 03:04:03 +02:00
Max	25c001c182	cluster/lib: implement simulacrum options	2024-08-12 03:04:03 +02:00
Max	d944dee3bc	WIP ENABLE DEBUG MODE	2024-08-12 02:56:57 +02:00
Max	aac5163a8b	cluster/lib: implement injectNixosConfigForServices to select individual services	2024-08-12 02:56:57 +02:00
Max	01c74f62cf	checks: add fake external storage module	2024-08-12 02:56:57 +02:00
Max	0110a4a0c3	checks: add a bunch of snakeoil keys	2024-08-12 02:56:57 +02:00