cluster/services/nextcloud: nextcloud29 -> nextcloud30

`flake.lock`: Update

.dvc/.gitignore

@@ -1,3 +0,0 @@
-/config.local
-/tmp
-/cache

.dvc/config

@@ -1,5 +0,0 @@
-[core]
-    remote = cdn
-['remote "cdn"']
-    url = s3://content-delivery/assets
-    endpointurl = https://object-storage.privatevoid.net

.dvcignore

@@ -1,3 +0,0 @@
-# Add patterns of files dvc should ignore, which could improve
-# the performance. Learn more at
-# https://dvc.org/doc/user-guide/dvcignore

.gitignore

@@ -2,4 +2,6 @@
 result
 result-*
 **/.direnv/
-.data/
+.data/
+.cache/
+.nixos-test-history

catalog/part.nix

@@ -0,0 +1,10 @@
+{ lib, ... }:
+
+{
+  perSystem = {
+    options.catalog = lib.mkOption {
+      type = with lib.types; lazyAttrsOf (lazyAttrsOf (lazyAttrsOf (submodule ./target.nix)));
+      default = {};
+    };
+  };
+}

catalog/target.nix

@@ -0,0 +1,31 @@
+{ lib, name, ... }:
+
+{
+  options = {
+    description = lib.mkOption {
+      type = lib.types.str;
+      default = name;
+    };
+
+    actions = lib.mkOption {
+      type = with lib.types; lazyAttrsOf (submodule {
+        options = {
+          description = lib.mkOption {
+            type = lib.types.str;
+            default = name;
+          };
+        
+          command = lib.mkOption {
+            type = lib.types.str;
+          };
+
+          packages = lib.mkOption {
+            type = with lib.types; listOf package;
+            default = [];
+          };
+        };
+      });
+      default = {};
+    };
+  };
+}

cluster/catalog/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./services.nix
+    ./secrets.nix
+  ];
+}

cluster/catalog/secrets.nix

@@ -0,0 +1,73 @@
+{ config, lib, withSystem, ... }:
+
+let
+  inherit (config) cluster hours;
+in
+
+{
+  perSystem = { config, pkgs, system, ... }: {
+    catalog.cluster = {
+      secrets = lib.pipe cluster.config.services [
+        (lib.mapAttrsToList (svcName: svcConfig: lib.mapAttrsToList (secretName: secretConfig: {
+          name = "${svcName}/${secretName}";
+          value = {
+            description = "Cluster secret '${secretName}' of service '${svcName}'";
+            actions = let
+              agenixRules = builtins.toFile "agenix-rules-shim.nix" /*nix*/ ''
+                builtins.fromJSON (builtins.readFile (builtins.getEnv "AGENIX_KEYS_JSON"))
+              '';
+
+              mkKeys = secretFile: nodes: builtins.toFile "agenix-keys.json" (builtins.toJSON {
+                "${secretFile}".publicKeys = (map (hour: hours.${hour}.ssh.id.publicKey) nodes) ++ cluster.config.secrets.extraKeys;
+              });
+
+              setupCommands = secretFile: nodes: let
+                agenixKeysJson = mkKeys secretFile nodes;
+              in ''
+                export RULES='${agenixRules}'
+                export AGENIX_KEYS_JSON='${agenixKeysJson}'
+                mkdir -p "$PRJ_ROOT/cluster/secrets"
+                cd "$PRJ_ROOT/cluster/secrets"
+              '';
+            in (lib.optionalAttrs (secretConfig.generate != null) {
+              generateSecret = {
+                description = "Generate this secret";
+                command = if secretConfig.shared then let
+                  secretFile = "${svcName}-${secretName}.age";
+                in ''
+                  ${setupCommands secretFile secretConfig.nodes}
+                  ${withSystem system secretConfig.generate} | agenix -e '${secretFile}'
+                '' else lib.concatStringsSep "\n" (map (node: let
+                  secretFile = "${svcName}-${secretName}-${node}.age";
+                in ''
+                  ${setupCommands secretFile [ node ]}
+                  ${withSystem system secretConfig.generate} | agenix -e '${secretFile}'
+                '') secretConfig.nodes);
+              };
+            }) // (if secretConfig.shared then let
+              secretFile = "${svcName}-${secretName}.age";
+            in {
+              editSecret = {
+                description = "Edit this secret";
+                command = ''
+                  ${setupCommands secretFile secretConfig.nodes}
+                  agenix -e '${secretFile}'
+                '';
+              };
+            } else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
+              secretFile = "${svcName}-${secretName}-${node}.age";
+            in {
+              description = "Edit this secret for '${node}'";
+              command = ''
+                ${setupCommands secretFile [ node ]}
+                agenix -e '${secretFile}'
+              '';
+            })));
+          };
+        }) svcConfig.secrets))
+        lib.concatLists
+        lib.listToAttrs
+      ];
+    };
+  };
+}

cluster/catalog/services.nix

@@ -0,0 +1,52 @@
+{ config, lib, ... }:
+
+let
+  inherit (config) cluster flake;
+in
+
+{
+  perSystem = { config, pkgs, ... }: {
+    catalog.cluster = {
+      services = lib.mapAttrs (name: svc: {
+        description = "Cluster service: ${name}";
+        actions = let
+          mkDeployAction = { description, agents }: {
+            inherit description;
+            packages = [
+              config.packages.cachix
+              pkgs.tmux
+            ];
+            command = let
+              cachixDeployJson = pkgs.writeText "cachix-deploy.json" (builtins.toJSON {
+                agents = lib.genAttrs agents (name: builtins.unsafeDiscardStringContext flake.nixosConfigurations.${name}.config.system.build.toplevel);
+              });
+            in ''
+              set -e
+              echo building ${toString (lib.length agents)} configurations in parallel
+              tmux new-session ${lib.concatStringsSep " split-window " (
+                map (host: let
+                  drvPath = builtins.unsafeDiscardStringContext flake.nixosConfigurations.${host}.config.system.build.toplevel.drvPath;
+                in '' 'echo building configuration for ${host}; nix build -L --no-link --store "ssh-ng://${host}" --eval-store auto "${drvPath}^*"'\; '') agents
+              )} select-layout even-vertical
+
+              source ~/.config/cachix/deploy
+              cachix deploy activate ${cachixDeployJson}
+              echo
+            '';
+          };
+        in {
+          deployAll = mkDeployAction {
+            description = "Deploy ALL groups of this service.";
+            agents = lib.unique (lib.concatLists (lib.attrValues svc.nodes));
+          };
+        } // lib.mapAttrs' (group: agents: {
+          name = "deployGroup-${group}";
+          value = mkDeployAction {
+            description = "Deploy the '${group}' group of this service.";
+            inherit agents;
+          };
+        }) svc.nodes;
+      }) cluster.config.services;
+    };
+  };
+}

cluster/default.nix

@@ -1,4 +1,4 @@
-{ lib, depot, hostName }:
+{ lib, depot }:
 
 lib.evalModules {
   specialArgs = {
@@ -7,16 +7,18 @@ lib.evalModules {
   modules = [
     # Arbitrary variables to reference across multiple services
     ./lib/vars
-    { vars = { inherit hostName; }; }
 
     # Cluster-level port-magic
     ../modules/port-magic
 
-    ../tools/inject.nix
     ./lib/services.nix
     ./lib/inject-nixos-config.nix
     ./lib/port-magic-multi.nix
+    ./lib/mesh.nix
+    ./lib/secrets.nix
+    ./lib/testing.nix
+    ./lib/lib.nix
 
     ./import-services.nix
   ];
-}
+}

cluster/inject.nix

@@ -1,15 +0,0 @@
-hostName:
-{ depot, lib, ... }:
-
-let
-  cluster = import ./. { inherit lib depot hostName; };
-in
-
-{
-  _module.args.cluster = {
-    inherit (cluster.config) vars;
-    inherit (cluster.config.vars) hosts;
-    inherit (cluster) config;
-  };
-  imports = cluster.config.out.injectedNixosConfig;
-}

cluster/lib/inject-nixos-config.nix

@@ -1,10 +1,10 @@
-{ lib, ... }:
+{ config, lib, ... }:
 with lib;
 
 {
-  options.out.injectedNixosConfig = mkOption {
-    description = "NixOS configuration modules to inject into the host.";
-    type = with types; listOf anything;
-    default = {};
+  options.out = mkOption {
+    description = "Output functions.";
+    type = with types; lazyAttrsOf (functionTo raw);
+    default = const [];
   };
 }

cluster/lib/lib.nix

@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+
+{
+  options.lib = {
+    forService = lib.mkOption {
+      description = "Enable these definitions for a particular service only.";
+      type = lib.types.functionTo lib.types.raw;
+      readOnly = true;
+      default = service: lib.mkIf (!config.simulacrum || lib.any (s: s == service) config.testConfig.activeServices);
+    };
+  };
+}

cluster/lib/mesh.nix

@@ -0,0 +1,17 @@
+{ config, lib, ... }:
+
+{
+  hostLinks = lib.pipe config.services [
+    (lib.filterAttrs (_: svc: svc.meshLinks != {}))
+    (lib.mapAttrsToList (svcName: svc:
+      lib.mapAttrsToList (groupName: links:
+        lib.genAttrs svc.nodes.${groupName} (hostName: lib.mapAttrs (_: cfg: { ... }: {
+          imports = [ cfg.link ];
+          ipv4 = config.vars.mesh.${hostName}.meshIp;
+        }) links)
+      ) svc.meshLinks
+    ))
+    (map lib.mkMerge)
+    lib.mkMerge
+  ];
+}

cluster/lib/secrets.nix

@@ -0,0 +1,14 @@
+{ lib, ... }:
+
+{
+  options.secrets = {
+    extraKeys = lib.mkOption {
+      type = with lib.types; listOf str;
+      description = "Additional keys with which to encrypt all secrets.";
+      default = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL5C7mC5S2gM0K6x0L/jNwAeQYbFSzs16Q73lONUlIkL max@TITAN"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMmdWfmAs/0rno8zJlhBFMY2SumnHbTNdZUXJqxgd9ON max@jericho"
+      ];
+    };
+  };
+}

cluster/lib/service-module.nix

@@ -1,14 +1,16 @@
-vars:
-{ config, lib, ... }:
+{ config, lib, name, ... }:
 with lib;
 
 let
-  notSelf = x: x != vars.hostName;
-
-  filterGroup = builtins.filter notSelf;
+  filterGroup = group: hostName: builtins.filter (x: x != hostName) group;
+  serviceName = name;
 in
 
 {
+  imports = [
+    ./services/secrets.nix
+  ];
+
   options = {
     nodes = mkOption {
       description = ''
@@ -21,12 +23,12 @@ in
           * X evaluators, Y smallBuilders, Z bigBuilders
         etc.
       '';
-      type = with types; attrsOf (oneOf [ str (listOf str) ]);
+      type = with types; lazyAttrsOf (oneOf [ str (listOf str) ]);
       default = [];
     };
     otherNodes = mkOption {
       description = "Other nodes in the group.";
-      type = with types; attrsOf (listOf str);
+      type = with types; lazyAttrsOf (functionTo (listOf str));
       default = [];
     };
     nixos = mkOption {
@@ -34,6 +36,36 @@ in
       type = with types; attrs;
       default = {};
     };
+    meshLinks = mkOption {
+      description = "Create host links on the mesh network.";
+      type = types.attrsOf (types.attrsOf (types.submodule {
+        options = {
+          link = mkOption {
+            type = types.deferredModule;
+            default = {};
+          };
+        };
+      }));
+      default = {};
+    };
+    simulacrum = {
+      enable = mkEnableOption "testing this service in the Simulacrum";
+      deps = mkOption {
+        description = "Other services to include.";
+        type = with types; listOf str;
+        default = [];
+      };
+      settings = mkOption {
+        description = "NixOS test configuration.";
+        type = types.deferredModule;
+        default = {};
+      };
+      augments = mkOption {
+        description = "Cluster augments (will be propagated).";
+        type = types.deferredModule;
+        default = {};
+      };
+    };
   };
-  config.otherNodes = builtins.mapAttrs (_: filterGroup) config.nodes;
+  config.otherNodes = builtins.mapAttrs (const filterGroup) config.nodes;
 }

cluster/lib/services.nix

@@ -2,18 +2,48 @@
 with lib;
 
 let
-  getHostConfigurations = svcConfig: hostName:
-    lib.mapAttrsToList (groupName: _: svcConfig.nixos.${groupName})
-    (lib.filterAttrs (_: lib.elem hostName) svcConfig.nodes);
+  getHostConfigurations = hostName: svcName: svcConfig: let
+    serviceConfigs =
+      lib.mapAttrsToList (groupName: _: svcConfig.nixos.${groupName})
+      (lib.filterAttrs (_: lib.elem hostName) svcConfig.nodes);
 
-  getServiceConfigurations = svcConfig: getHostConfigurations svcConfig config.vars.hostName;
+    secretsConfig = let
+      secrets = lib.filterAttrs (_: secret: lib.any (node: node == hostName) secret.nodes) svcConfig.secrets;
+    in {
+      age.secrets = lib.mapAttrs' (secretName: secretConfig: {
+        name = "cluster-${svcName}-${secretName}";
+        value = {
+          inherit (secretConfig) path mode owner group;
+          file = ../secrets/${svcName}-${secretName}${lib.optionalString (!secretConfig.shared) "-${hostName}"}.age;
+        };
+      }) secrets;
+
+      systemd.services = lib.mkMerge (lib.mapAttrsToList (secretName: secretConfig: lib.genAttrs secretConfig.services (systemdServiceName: {
+        restartTriggers = [ "${../secrets/${svcName}-${secretName}${lib.optionalString (!secretConfig.shared) "-${hostName}"}.age}" ];
+      })) secrets);
+    };
+  in serviceConfigs ++ [
+    secretsConfig
+  ];
+
+  introspectionModule._module.args.cluster = {
+    inherit (config) vars;
+    inherit config;
+  };
 in
 
 {
   options.services = mkOption {
     description = "Cluster services.";
-    type = with types; attrsOf (submodule (import ./service-module.nix config.vars));
+    type = with types; attrsOf (submodule ./service-module.nix);
     default = {};
   };
-  config.out.injectedNixosConfig = lib.flatten (lib.mapAttrsToList (_: getServiceConfigurations) config.services);
+
+  config.out = {
+    injectNixosConfigForServices = services: hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) (lib.getAttrs services config.services))) ++ [
+      introspectionModule
+    ];
+
+    injectNixosConfig = config.out.injectNixosConfigForServices (lib.attrNames config.services);
+  };
 }

cluster/lib/services/secrets.nix

@@ -0,0 +1,57 @@
+{ lib, name, ... }:
+
+let
+  serviceName = name;
+in
+
+{
+  options.secrets = lib.mkOption {
+    type = lib.types.lazyAttrsOf (lib.types.submodule ({ config, name, ... }: {
+      options = {
+        shared = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Whether this secret should be the same on all nodes.";
+        };
+
+        nodes = lib.mkOption {
+          type = with lib.types; listOf str;
+          default = [ ];
+        };
+
+        generate = lib.mkOption {
+          type = with lib.types; nullOr (functionTo str);
+          description = "Command used to generate this secret.";
+          default = null;
+        };
+
+        path = lib.mkOption {
+          type = lib.types.path;
+          default = "/run/agenix/cluster-${serviceName}-${name}";
+        };
+
+        mode = lib.mkOption {
+          type = lib.types.str;
+          default = "0400";
+        };
+
+        owner = lib.mkOption {
+          type = lib.types.str;
+          default = "root";
+        };
+
+        group = lib.mkOption {
+          type = lib.types.str;
+          default = "root";
+        };
+
+        services = lib.mkOption {
+          type = with lib.types; listOf str;
+          description = "Services to restart when this secret changes.";
+          default = [];
+        };
+      };
+    }));
+    default = {};
+  };
+}

cluster/lib/testing.nix

@@ -0,0 +1,15 @@
+{ lib, ... }:
+
+{
+  options = {
+    simulacrum = lib.mkOption {
+      description = "Whether we are in the Simulacrum.";
+      type = lib.types.bool;
+      default = false;
+    };
+    testConfig = lib.mkOption {
+      type = lib.types.attrs;
+      readOnly = true;
+    };
+  };
+}

cluster/part.nix

@@ -0,0 +1,16 @@
+{ depot, lib, ... }:
+
+{
+  imports = [
+    ./catalog
+    ./simulacrum/checks.nix
+  ];
+
+  options.cluster = lib.mkOption {
+    type = lib.types.raw;
+  };
+
+  config.cluster = import ./. {
+    inherit depot lib;
+  };
+}

cluster/secrets/attic-serverToken.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> ssh-ed25519 d3WGuA ZLjCSe5wrN6abvvRmQjE+VXtRr+avP/CLPD7djXNr0M
+g8i9ambJGL2Q+ZLB6c6MxV9BryAgX4qZctJ9qByJ4n8
+-> ssh-ed25519 P/nEqQ zSGcZuufOAnTkPr74ZjwyISdLlfxBxqgmyWivxq1/Uo
+gArusBfIfsZ5/gwMYHLzDHTbgVGWDttbi0IAhvclRO4
+-> ssh-ed25519 YIaSKQ J4Fy0VSjdMPRgzysQptIUKiRR0TAgu0q1BYhtIpGkWU
+kKzmF3OUbGU40d33R15nMraUDZiFRoz9Z00XjjSk9Jw
+-> ssh-ed25519 NO562A BNQV8JodzTiNs/V+rFQxcsrhKJ3nRIFtWk6VxHzCRio
+ZyauAdOrPbADSDdBQoB+39MB2r7Ro4d0XwZIjf2z9Jo
+-> ssh-ed25519 5/zT0w hdMuyOmNKTlMKPn4w9VQFVXZkJNm1XSPAZ/Zip5WW04
+wcnur+BRQPqKzpV3vl7pn1VIGRK3GxQEUaQIefrZuI4
+--- 5AdxXgFmDm2w012QjpJ3gqlbfvkPm8fkEJjm8kV18G0
+&Ãf§äIT¼-ÿY!ŒÍ,Vu<56>Â9õÿöBFrœŠ´½4–ù™BÕÝ/®UäH˜rþ¸ž	#ƒˆç
ÄÝÕº†®UóQ¢ÿŽx$G{ÅŠMà2¡^/˜§¥Éè?12É¿t1©¿í¸&[}nêDAÛlýÑýˆ8uG®éZŽ×b¯èàîåd:@ÿ!Õþ
jîƒÚáÈNµrâlA³~

cluster/secrets/hercules-ci-multi-agent-cacheCredentials-prophet.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A jNUNRaIQC1DUBiacnWc3xjMUAxnAgiyJhRA74cof3Ec
+oZZq1AQ3F0nvrk7KpinLWgT3cIoCYZ5R1s0us69OI8E
+-> ssh-ed25519 5/zT0w FmoxaTg75/xaDmSOHL5Xs6QOr5rLG/sr5TmPMfkOqxw
+XXQrFxxt5GOzXgJoPY8U71NSYi/IWmL3QrenvOuQ43Q
+-> ssh-ed25519 YIaSKQ ++dqG+dr8ie+4sPW7L+eVkXvOVvM+/oBR722S2sQsSg
+879pmnhOtZ/MiMUwDlyujykQXNmCepI2FSU2QcvvkrA
+--- QcvlVdv2fYMKmT/aCpTjdmGJ+9KnUvZCZNtl7WhgCbw
+ï!jÊwŸ~×f%ÝJ>˜H	³·2ü9¬¥.VhþÅ·<C385>²«O!$iÄ<>ÝžÔ<>4\_̆J¸„šÀT>²J£‘î8Y´\ÁI³kÕýïk—tŒG(ÃAO¦#Ùš“#Ü(·LœøÍáô’0éh=[ÈRîw•Uj¸iVý2ÁÕ(ìÊBGgÔ„^L7fÍÊ«"zVµ<56>nË)ÑõË÷9½ï›<IäõúÃÍw1Š

cluster/secrets/hercules-ci-multi-agent-clusterJoinToken-hyprspace-prophet.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A eB0Rkoz721eI1UlyAhHWIrBnTEFoh6z3UL24EljaNzA
+dNsoal+y68XM4HXRyg1PUmrWilW1n3h78TmTcqHFEjc
+-> ssh-ed25519 5/zT0w SF16JelBZe0vZtzNEHiEfprJOqzoyxhTH3ldQdbo5wE
+95wJNWQEGqHj4Pknnk1RrgWPOqZOhlNsSvFTv8rfc08
+-> ssh-ed25519 YIaSKQ 68vS4sQGTDEaTVVxfs/xeTv379MQ3JE7iyLb1PbUuis
+1Bh53X0QFednXw74lQ+FbqNDkLBra9rx6nOybcD3FiQ
+--- HIcPirpTTtlUUGEemDXND/nwiWs4BEhM4rYX18mx71E
+箜_Ÿvw©\ˆ¯j2æVrK(™á2åÚ@ξ€;Y®AQAƒlMÛá[ÙÁW â—ßƀы<v#"ùóBŒ’O€™É^†©¦-­ø¡+ž*m}›¦<>ª\“ª¡gÒ¹'kÓ2I~T¾w’M|¼jó¬˜+*BÖ%æ°xx‘‘€Ó¸õ{Ž O™;Fd„M“
+ ÝPÙEB¡mãdBý¡¿¨[•¼í
5Þf˜‰ü#öL- ¢³.4gŽ”FnÀ£q¬òv<C3B2>SV¹¥°÷¤êYÉkä·ï@ÓçlRn
+!¸'mÿSGìqóÊÖ“0dY1ïL!Jñðä üIÿw

cluster/secrets/idm-serviceAccountCredentials-grail.age

@@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A 2Su0u03W90TKuR0wg1/dcokIbTzO5eATBmkFPyqfJG0
+IhBAWy5YYFFOqG9hc+AkVrKewTls84SFV9Kz/lOTV2U
+-> ssh-ed25519 5/zT0w YsyFCW1FsiGwiYJNYCITlLWk6Y5dR3K5v+gJqlsWQTg
+vtR1GCT2zrHNco/yPvMqQmlPyDja53lSRsO1DmnCSlo
+-> ssh-ed25519 P/nEqQ c8l4fOuvZn9V8+6vpRpGNGldEi4iA+5qVg1B+jArU1w
+zgS0urO8MZYo8cZq5Nz/R1x9cZ0vZgppJx6X5UecJ0s
+-> ?^lS,zDo-grease ^ZMN! V*+oK^9 GyJ[
+ZATLlHQ+kFjStI2ykQXq+KhvAR+XeW+POj6cJ59awzpMwq8JGbyaE1m5Cq8XA6u3
+xFE6
+--- 3JfCfv5CJYKGuvnbvoymhHhfkM99NkYMdGOL3Xicga8
+ðíçqÂ`ë#Ññ›„oq6üÄÑZÃõ˜<>Žh
$wH"©läNøØ£¨IÛL3ä¯uY‹WŽœ<11>›T¹À*G<>GÂx¦nD2IÈ
ù«y+]ßT{gäð©<C3B0>ìÓinœÖÈçßEa¥ìœk¸zοP	”M…

cluster/secrets/patroni-PATRONI_REPLICATION_PASSWORD.age

@@ -0,0 +1,15 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A o0R34LvRy19TseKFBi6iJVJSpuPWamIlL1UnX95+yVU
+9yjfDbf7J9q/L2Z8OkFlOcniYNfO9YJBdtNkLyQAzF4
+-> ssh-ed25519 5/zT0w AqcfbKIO1vE0TjkDvZOkCcMeRCz5ATfQZyoKecoDWQE
+beYLRlS/ZzteQ1MNhyGuIenuEHSRqkzYJRasomThBLU
+-> ssh-ed25519 FfIUuQ 9JeHQPQgOYSzA2cjR6jwisZYPRRYGQMSyOW49LVEo30
+TAd1otmjEo1CvOVX3gZe2rk6Rk/IEjF2DllpQ9+o6ak
+-> ssh-ed25519 d3WGuA 1RNgW2d+Nh66Li4hAsP7/3ATZsqHWL+9pwNqPej1ykw
+tN6e8kBNz4tknvWBrVoQ6nABbeglac6AVUlz32ZFMzA
+-> ssh-ed25519 P/nEqQ oHqCnNvCWIprBbVy0w2LdHBaJllbNwD+WbdAgIFrhww
+6Dgnv/HyYaFzAiRUySqYVjBPScLMK8M9Roo8UCqMGEM
+-> 4Vf|uj93-grease x5,kZ; -xgrjD8
+6Gw1SIrH9N0GVksuPQm46tD/7Ckn6vkG5Z9CDhu4hj4YO1X8ug
+--- eo6VHBS0rJXNXA4MFGBtVfJRQ7hNNJ7PMeMjvE1Hms8
+‘7<EFBFBD>¸ATº<>ÖŸ@OXåø?$ýÛ“XeÞ€<>{T|P†.3;EºÌ3mLÛã"o“´"õcèí—”#ü,"Í¥CtÒô½;¥ÂˆÒ³IÚR	FOócD"âúK;¯{HÛÝ×ký™.d[sƒ·/¼R!à‹vk.®®W–‘°ñÿãºóç×<C3A7>ƒ6{Íþ
°òn<C3B2>­È_M,¶½¬6o`Óô£?×@…ŠRX¨ñù´€()É<>UPëâ	o9qÙFJûˆÆ’ÂúDkŒ#‚{D‰+[pÞu½õÌkúÊÎlMVêm+™ kDiŸ­‡ó”l¤œûT=·ji6.ªUS¦–ö³óŽ\Æ-s€¦b«!eɳ‰:¿/—°NgŒSï—«¸

cluster/secrets/wireguard-meshPrivateKey-grail.age

@@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A BNIU8M5X5C4LSiie6S4zVraFQAsyGKAv7BwLVIXHiFM
+LLcXZ7tiTUnN+tJLwqqs1hLZ8usCDWqNVGr1lAn5OQs
+-> ssh-ed25519 5/zT0w H/SGf0oYVg/JCd07bicWL1LWQwExr0gbi+gV1j7Fy2M
+yHjguPtS8ItpY+pAR3lLVpXQxq7d3cuQYU5DHs2qjMc
+-> ssh-ed25519 P/nEqQ z1us0mTbOuLrkI7n6doG+JVFAuqwZvC0dEfdGauM+Fg
+P/tKnt5gZ66HAWR0/pqpmJMHp6hLbcjwE3BhO9NCkZY
+-> ((I-grease
+r66LwGiqumMp/NlcnLgOaxZ7cfQMBCr4Rq9aJdjUck69113hNf4orC/bGVCDhmdu
+s1cSHPVw1hys
+--- FxWSO98U5IDaGPs57hzO70gVN/ELN0/UxKKmIoxadks
+1ÊnûEHvóî_QíÄV†7¬Çæ•Ãܲé¶m¡z2'ÛÎ¥¯zWÚ)¼Ôç.»!ãi#¬TXÎT‰k[Fy
üˆEë!>á¨tÁ !‹‚*Ã

cluster/services/acme-client/augment.nix

@@ -0,0 +1,60 @@
+{ config, pkgs, ... }:
+
+let
+  lift = config;
+in
+
+{
+  nowhere.names = {
+    "acme-v02.api.letsencrypt.org" = "stepCa";
+    "api.buypass.com" = "stepCa";
+  };
+
+  nodes.nowhere = { config, ... }: {
+    links.stepCa.protocol = "https";
+
+    environment.etc.step-ca-password.text = "";
+
+    services = {
+      step-ca = {
+        enable = true;
+        address = config.links.stepCa.ipv4;
+        inherit (config.links.stepCa) port;
+        intermediatePasswordFile = "/etc/step-ca-password";
+        settings = {
+          root = "${lift.nowhere.certs.ca}/ca.pem";
+          crt = "${lift.nowhere.certs.intermediate}/cert.pem";
+          key = "${lift.nowhere.certs.intermediate}/cert-key.pem";
+          address = config.links.stepCa.tuple;
+          db = {
+            type = "badgerv2";
+            dataSource = "/var/lib/step-ca/db";
+          };
+          authority.provisioners = [
+            {
+              type = "ACME";
+              name = "snakeoil";
+              challenges = [
+                "dns-01"
+                "http-01"
+              ];
+            }
+          ];
+        };
+      };
+
+      nginx.virtualHosts = {
+        "acme-v02.api.letsencrypt.org".locations."/".extraConfig = ''
+          rewrite /directory /acme/snakeoil/directory break;
+        '';
+        "api.buypass.com".locations."/".extraConfig = ''
+          rewrite /acme/directory /acme/snakeoil/directory break;
+        '';
+      };
+    };
+  };
+
+  defaults.environment.etc."dummy-secrets/acmeDnsApiKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
+  defaults.environment.etc."dummy-secrets/acmeDnsDirectKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
+  defaults.environment.etc."dummy-secrets/acmeDnsDbCredentials".text = "PGPASSWORD=simulacrum";
+}

cluster/services/acme-client/client.nix

@@ -1,10 +1,82 @@
-{ cluster, config, pkgs, ... }:
+{ cluster, config, depot, lib, pkgs, ... }:
+
+let
+  authoritativeServers = map
+    (node: cluster.config.hostLinks.${node}.dnsAuthoritative.tuple)
+    cluster.config.services.dns.nodes.authoritative;
+
+  execScript = pkgs.writeShellScript "acme-dns-exec" ''
+    action="$1"
+    subdomain="''${2%.${depot.lib.meta.domain}.}"
+    key="$3"
+    umask 77
+    source "$EXEC_ENV_FILE"
+    headersFile="$(mktemp)"
+    echo "X-Direct-Key: $ACME_DNS_DIRECT_STATIC_KEY" > "$headersFile"
+    case "$action" in
+      present)
+        for i in {1..5}; do
+          ${pkgs.curl}/bin/curl -X POST -s -f -H "@$headersFile" \
+            "${cluster.config.links.acmeDnsApi.url}/update" \
+            --data '{"subdomain":"'"$subdomain"'","txt":"'"$key"'"}' && break
+          sleep 5
+        done
+        ;;
+    esac
+  '';
+in
 
 {
-  age.secrets.pdns-api-key-acme = cluster.config.vars.pdns-api-key-secret // { owner = "acme"; };
+  age.secrets.acmeDnsApiKey = {
+    file = ../dns/acme-dns-direct-key.age;
+    owner = "acme";
+  };
 
-  security.acme.defaults.credentialsFile = pkgs.writeText "acme-pdns-credentials" ''
-    PDNS_API_URL=${cluster.config.links.powerdns-api.url}
-    PDNS_API_KEY_FILE=${config.age.secrets.pdns-api-key-acme.path}
-  '';
+  security.acme.acceptTerms = true;
+  security.acme.maxConcurrentRenewals = 0;
+  security.acme.defaults = {
+    email = depot.lib.meta.adminEmail;
+    extraLegoFlags = lib.flatten [
+      (map (x: [ "--dns.resolvers" x ]) authoritativeServers)
+      "--dns-timeout" "30"
+    ];
+    credentialsFile = pkgs.writeText "acme-exec-config" ''
+      EXEC_PATH=${execScript}
+      EXEC_ENV_FILE=${config.age.secrets.acmeDnsApiKey.path}
+    '';
+  };
+
+  systemd.services = lib.mapAttrs' (name: value: {
+    name = "acme-${name}";
+    value = {
+      distributed.enable = value.dnsProvider != null;
+      preStart = let
+        serverList = lib.pipe authoritativeServers [
+          (map (x: "@${x}"))
+          (map (lib.replaceStrings [":53"] [""]))
+          lib.escapeShellArgs
+        ];
+        domainList = lib.pipe ([ value.domain ] ++ value.extraDomainNames) [
+          (map (x: "${x}."))
+          (map (lib.replaceStrings ["*"] ["x"]))
+          lib.unique
+          lib.escapeShellArgs
+        ];
+      in ''
+        echo Testing availability of authoritative DNS servers
+        for i in {1..60}; do
+          ${pkgs.dig}/bin/dig +short ${serverList} ${domainList} >/dev/null && break
+          echo Retry [$i/60]
+          sleep 10
+        done
+        echo Available
+      '';
+      serviceConfig = {
+        Restart = "on-failure";
+        RestartMaxDelaySec = 30;
+        RestartSteps = 5;
+        RestartMode = "direct";
+      };
+    };
+  }) config.security.acme.certs;
 }

cluster/services/acme-client/default.nix

@@ -1,6 +1,7 @@
 {
   services.acme-client = {
-    nodes.client = [ "checkmate" "thunderskin" "VEGAS" "prophet" ];
+    nodes.client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
     nixos.client = ./client.nix;
+    simulacrum.augments = ./augment.nix;
   };
 }

cluster/services/attic/attic-server-token.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A mLMev+YA6zSxAWIIlwseZk8Skl4hfNNsWQPmLV7DxTo
-AEi55ZXzyYbZyludcP5Yywx7QDgFODh6z8+M2nxMAl4
--> ssh-ed25519 5/zT0w 91baPvXx4UdmyYCCIqc1M+Cb7pqdSx3/cgmfuexeUgY
-kePQp8flAsXPMLxJiQPoJuHEPPI+FzaSF+VL9U9jhwI
--> ssh-ed25519 d3WGuA U8Q68hN+5fI4xto/lpCiVyts00ezCzftfLvFFew7aiY
-B4wv05Y2gpl5qjV1Rbc6JSJk3coN6TFMB5FspwzLnlI
--> :0eX-grease
-ghW6iCUZj0e04I8Ba3CHzg
---- aHnzzTi1WxtHXGcQO1hNgmy04wyyObmYBcSq5fmbnAg
-Ñdï<EFBFBD>ÎÁŽ#¹<>¬sä®nƒŒó¤ž§–F#5IZ<49><5A>¯áË2wb®×¨âÑÞüË›oœkm÷ÒåN&"¤ü0LeÑzIjx—µzxF€´>ršúEq´Ý¤¥A‘nx¿šB!@‰ÕŸÆò2r©:ïm5í-Xl5çAåð*ëÌSV¿R3`Ð艨{ÿò<C3BF>ï©#ÍJgHÖ‡ÊÉg

cluster/services/attic/binary-cache.nix

@@ -1,13 +1,16 @@
-{ config, tools, ... }:
-with tools.nginx;
-let
-  addrSplit' = builtins.split ":" config.services.minio.listenAddress;
-  addrSplit = builtins.filter builtins.isString addrSplit';
-  host' = builtins.head addrSplit;
-  host = if host' == "" then "127.0.0.1" else host';
-  port = builtins.head (builtins.tail addrSplit);
-in
+{ config, cluster, depot, lib, ... }:
+with depot.lib.nginx;
 {
+  links = {
+    atticNixStoreInternalRedirect.protocol = "http";
+    garageNixStoreInternalRedirect.protocol = "http";
+  };
+
+  security.acme.certs."cache.${depot.lib.meta.domain}" = {
+    dnsProvider = "exec";
+    webroot = lib.mkForce null;
+  };
+
   services.nginx.upstreams = {
       nar-serve.extraConfig = ''
       random;
@@ -15,40 +18,73 @@ in
       server ${config.links.nar-serve-nixos-org.tuple} fail_timeout=0;
     '';
     nix-store.servers = {
-      "${config.links.atticServer.tuple}" = {
+      "${config.links.garageNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
       };
-      "${host}:${port}" = {
+      "${config.links.atticNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
-        backup = true;
       };
     };
   };
   services.nginx.appendHttpConfig = ''
     proxy_cache_path /var/cache/nginx/nixstore levels=1:2 keys_zone=nixstore:10m max_size=10g inactive=24h use_temp_path=off;
   '';
-  services.nginx.virtualHosts."cache.${tools.meta.domain}" = vhosts.basic // {
-    locations = {
-      "= /".return = "302 /404";
-      "/" = {
-        proxyPass = "http://nix-store/nix-store$request_uri";
-        extraConfig = ''
-          proxy_next_upstream error http_500 http_404;
-        '';
+  services.nginx.virtualHosts = {
+    "cache.${depot.lib.meta.domain}" = vhosts.basic // {
+      locations = {
+        "= /".return = "302 /404";
+        "/" = {
+          proxyPass = "http://nix-store";
+          extraConfig = ''
+            proxy_next_upstream error http_500 http_502 http_404;
+          '';
+        };
+        "/nix/store" = {
+          proxyPass = "http://nar-serve";
+          extraConfig = ''
+            proxy_next_upstream error http_500 http_404;
+          '';
+        };
       };
-      "/nix/store" = {
-        proxyPass = "http://nar-serve";
+      extraConfig = ''
+        proxy_cache nixstore;
+        proxy_cache_use_stale error timeout http_500 http_502;
+        proxy_cache_lock on;
+        proxy_cache_key $request_uri;
+        proxy_cache_valid 200 24h;
+      '';
+    };
+    "garage-nix-store.internal.${depot.lib.meta.domain}" = {
+      serverName = "127.0.0.1";
+      listen = [
+        {
+          addr = "127.0.0.1";
+          inherit (config.links.garageNixStoreInternalRedirect) port;
+        }
+      ];
+      locations."/" = {
+        proxyPass = with cluster.config.links.garageWeb; "${protocol}://nix-store.${hostname}";
+        recommendedProxySettings = false;
         extraConfig = ''
-          proxy_next_upstream error http_500 http_404;
+          proxy_set_header Host "nix-store.${cluster.config.links.garageWeb.hostname}";
+        '';
+      };
+    };
+    "attic-nix-store.internal.${depot.lib.meta.domain}" = {
+      serverName = "127.0.0.1";
+      listen = [
+        {
+          addr = "127.0.0.1";
+          inherit (config.links.atticNixStoreInternalRedirect) port;
+        }
+      ];
+      locations."/" = {
+        proxyPass = "https://cache-api.${depot.lib.meta.domain}/nix-store$request_uri";
+        recommendedProxySettings = false;
+        extraConfig = ''
+          proxy_set_header Host "cache-api.${depot.lib.meta.domain}";
         '';
       };
     };
-    extraConfig = ''
-      proxy_cache nixstore;
-      proxy_cache_use_stale error timeout http_500 http_502;
-      proxy_cache_lock on;
-      proxy_cache_key $request_uri;
-      proxy_cache_valid 200 24h;
-    '';
   };
 }

cluster/services/attic/default.nix

@@ -1,14 +1,60 @@
+{ config, depot, ... }:
+
 {
   services.attic = {
     nodes = {
-      server = [ "VEGAS" ];
+      monolith = [ "VEGAS" "prophet" ];
+      server = [ "VEGAS" "grail" "prophet" ];
     };
     nixos = {
+      monolith = [
+        ./server.nix
+      ];
       server = [
         ./server.nix
         ./binary-cache.nix
         ./nar-serve.nix
       ];
     };
+    meshLinks.server.attic.link.protocol = "http";
+    secrets = let
+      inherit (config.services.attic) nodes;
+    in {
+      serverToken = {
+        nodes = nodes.server;
+      };
+      dbCredentials = {
+        nodes = nodes.server;
+        owner = "atticd";
+      };
+    };
+  };
+
+  garage = config.lib.forService "attic" {
+    keys.attic.locksmith = {
+      nodes = config.services.attic.nodes.server;
+      owner = "atticd";
+      format = "aws";
+    };
+    buckets.attic = {
+      allow.attic = [ "read" "write" ];
+    };
+  };
+
+  dns.records = let
+    serverAddrs = map
+      (node: depot.hours.${node}.interfaces.primary.addrPublic)
+      config.services.attic.nodes.server;
+  in config.lib.forService "attic" {
+    cache.target = serverAddrs;
+  };
+
+  ways = config.lib.forService "attic" {
+    cache-api = {
+      consulService = "atticd";
+      extras.extraConfig = ''
+        client_max_body_size 4G;
+      '';
+    };
   };
 }

cluster/services/attic/nar-serve.nix

@@ -1,4 +1,4 @@
-{ config, depot, tools, ... }:
+{ config, depot, ... }:
 
   let
     mkNarServe = NAR_CACHE_URL: PORT: {
@@ -17,6 +17,6 @@
     nar-serve-nixos-org.protocol = "http";
   };
 
-  systemd.services.nar-serve-self = mkNarServe "https://cache.${tools.meta.domain}" config.links.nar-serve-self.portStr;
+  systemd.services.nar-serve-self = mkNarServe "https://cache.${depot.lib.meta.domain}" config.links.nar-serve-self.portStr;
   systemd.services.nar-serve-nixos-org = mkNarServe "https://cache.nixos.org" config.links.nar-serve-nixos-org.portStr;
 }

cluster/services/attic/server.nix

@@ -1,44 +1,47 @@
-{ config, depot, lib, tools, ... }:
+{ cluster, config, depot, lib, ... }:
 
 let
-  dataDir = "/srv/storage/private/attic";
+  inherit (cluster.config.services.attic) secrets;
+
+  link = cluster.config.hostLinks.${config.networking.hostName}.attic;
+
+  isMonolith = lib.elem config.networking.hostName cluster.config.services.attic.nodes.monolith;
 in
 
 {
-  imports = [
-    depot.inputs.attic.nixosModules.atticd
-  ];
-
-  ascensions.attic-standalone = {
-    requiredBy = [ "attic.service" ];
-    before = [ "attic.service" ];
-    incantations = i: [ ];
-  };
-
-  age.secrets.atticServerToken.file = ./attic-server-token.age;
-
-  links.atticServer.protocol = "http";
+  services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];
 
   services.atticd = {
     enable = true;
+    package = depot.inputs.attic.packages.attic-server;
 
-    credentialsFile = config.age.secrets.atticServerToken.path;
+    environmentFile = secrets.serverToken.path;
+    mode = if isMonolith then "monolithic" else "api-server";
 
     settings = {
-      listen = config.links.atticServer.tuple;
+      listen = link.tuple;
 
       chunking = {
-        nar-size-threshold = 512 * 1024;
-        min-size = 64 * 1024;
-        avg-size = 512 * 1024;
-        max-size = 1024 * 1024;
+        nar-size-threshold = 0;
+        min-size = 0;
+        avg-size = 0;
+        max-size = 0;
       };
 
-      database.url = "sqlite://${dataDir}/server.db?mode=rwc";
+      compression.type = "none";
+
+      database.url = "postgresql://attic@${cluster.config.links.patroni-pg-access.tuple}/attic";
 
       storage = {
-        type = "local";
-        path = "${dataDir}/chunks";
+        type = "s3";
+        region = "us-east-1";
+        endpoint = cluster.config.links.garageS3.url;
+        bucket = "attic";
+      };
+
+      garbage-collection = {
+        interval = "2 weeks";
+        default-retention-period = "3 months";
       };
     };
   };
@@ -47,20 +50,44 @@ in
     users.atticd = {
       isSystemUser = true;
       group = "atticd";
-      home = dataDir;
+      home = "/var/lib/atticd";
       createHome = true;
     };
     groups.atticd = {};
   };
 
-  systemd.services.atticd.serviceConfig = {
-    DynamicUser = lib.mkForce false;
-    ReadWritePaths = [ dataDir ];
+  systemd.services.atticd = {
+    after = [ "postgresql.service" ];
+    distributed = lib.mkIf isMonolith {
+      enable = true;
+      registerService = "atticd";
+    };
+    serviceConfig = {
+      DynamicUser = lib.mkForce false;
+      RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
+      SystemCallFilter = lib.mkAfter [ "@resources" ];
+    };
+    environment = {
+      AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
+      PGPASSFILE = secrets.dbCredentials.path;
+    };
   };
 
-  services.nginx.virtualHosts."cache-api.${tools.meta.domain}" = tools.nginx.vhosts.proxy config.links.atticServer.url // {
-    extraConfig = ''
-      client_max_body_size 4G;
-    '';
+  consul.services.atticd = {
+    mode = if isMonolith then "manual" else "direct";
+    definition = {
+      name = "atticd";
+      id = "atticd-${config.services.atticd.mode}";
+      address = link.ipv4;
+      inherit (link) port;
+      checks = [
+        {
+          name = "Attic Server";
+          id = "service:atticd:backend";
+          interval = "5s";
+          http = link.url;
+        }
+      ];
+    };
   };
 }

cluster/services/bitwarden/default.nix

@@ -0,0 +1,10 @@
+{ depot, ... }:
+
+{
+  services.bitwarden = {
+    nodes.host = [ "VEGAS" ];
+    nixos.host = ./host.nix;
+  };
+
+  dns.records.keychain.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ];
+}

cluster/services/bitwarden/host.nix

@@ -1,5 +1,5 @@
-{ config, lib, tools, ... }:
-with tools.nginx;
+{ config, lib, depot, ... }:
+with depot.lib.nginx;
 {
   links.bitwarden.protocol = "http";
 

cluster/services/cachix-deploy-agent/agent.nix

@@ -1,10 +1,9 @@
-{ config, ... }:
+{ cluster, depot, ... }:
 
 {
-  age.secrets.cachixDeployToken.file = ./credentials/${config.networking.hostName}.age;
-
   services.cachix-agent = {
     enable = true;
-    credentialsFile = config.age.secrets.cachixDeployToken.path;
+    credentialsFile = cluster.config.services.cachix-deploy-agent.secrets.token.path;
+    package = depot.packages.cachix;
   };
 }

cluster/services/cachix-deploy-agent/default.nix

@@ -1,6 +1,10 @@
 {
-  services.cachix-deploy-agent = {
-    nodes.agent = [ "checkmate" "prophet" "VEGAS" "thunderskin" ];
+  services.cachix-deploy-agent = { config, ... }: {
+    nodes.agent = [ "checkmate" "grail" "prophet" "VEGAS" "thunderskin" ];
     nixos.agent = ./agent.nix;
+    secrets.token = {
+      nodes = config.nodes.agent;
+      shared = false;
+    };
   };
 }

cluster/services/cdn-shield/default.nix

@@ -0,0 +1,12 @@
+{ depot, ... }:
+
+{
+  dns.records = let
+    cdnShieldAddr = [ depot.hours.VEGAS.interfaces.primary.addrPublic ];
+  in {
+    "fonts-googleapis-com.cdn-shield".target = cdnShieldAddr;
+    "fonts-gstatic-com.cdn-shield".target = cdnShieldAddr;
+    "cdnjs-cloudflare-com.cdn-shield".target = cdnShieldAddr;
+    "wttr-in.cdn-shield".target = cdnShieldAddr;
+  };
+}

cluster/services/certificates/default.nix

@@ -1,7 +1,7 @@
 {
   services.certificates = {
     nodes = {
-      internal-wildcard = [ "checkmate" "thunderskin" "VEGAS" "prophet" ];
+      internal-wildcard = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
     };
     nixos = {
       internal-wildcard = [

cluster/services/certificates/internal-wildcard.nix

@@ -1,7 +1,7 @@
-{ config, lib, pkgs, tools, ... }:
+{ config, lib, pkgs, depot, ... }:
 
 let
-  inherit (tools.meta) domain;
+  inherit (depot.lib.meta) domain;
 
   extraGroups = [ "nginx" ]
     ++ lib.optional config.services.kanidm.enableServer "kanidm";
@@ -11,12 +11,12 @@ in
   security.acme.certs."internal.${domain}" = {
     domain = "*.internal.${domain}";
     extraDomainNames = [ "*.internal.${domain}" ];
-    dnsProvider = "pdns";
+    dnsProvider = "exec";
     group = "nginx";
     postRun = ''
-      ${pkgs.acl}/bin/setfacl -Rb out/
+      ${pkgs.acl}/bin/setfacl -Rb .
       ${lib.concatStringsSep "\n" (
-        map (group: "${pkgs.acl}/bin/setfacl -Rm g:${group}:rX out/") extraGroups
+        map (group: "${pkgs.acl}/bin/setfacl -Rm g:${group}:rX .") extraGroups
       )}
     '';
   };

cluster/services/chant/default.nix

@@ -0,0 +1,11 @@
+{ config, ... }:
+
+{
+  services.chant = {
+    nodes.listener = config.services.consul.nodes.agent;
+    nixos.listener = [
+      ./listener.nix
+    ];
+    simulacrum.deps = [ "consul" ];
+  };
+}

cluster/services/chant/listener.nix

@@ -0,0 +1,82 @@
+{ config, lib, pkgs, ... }:
+
+let
+  consul = config.links.consulAgent;
+
+  validTargets = lib.pipe config.systemd.services [
+    (lib.filterAttrs (name: value: value.chant.enable))
+    lib.attrNames
+  ];
+
+  validTargetsJson = pkgs.writeText "chant-targets.json" (builtins.toJSON validTargets);
+
+  eventHandler = pkgs.writers.writePython3 "chant-listener-event-handler" {
+    flakeIgnore = [ "E501" ];
+  } ''
+    import json
+    import sys
+    import os
+    import subprocess
+    import base64
+
+    validTargets = set()
+    with open("${validTargetsJson}", "r") as f:
+        validTargets = set(json.load(f))
+
+    events = json.load(sys.stdin)
+
+    cacheDir = os.getenv("CACHE_DIRECTORY", "/var/cache/chant")
+
+    indexFile = f"{cacheDir}/index"
+
+    oldIndex = "old-index"
+    if os.path.isfile(indexFile):
+        with open(indexFile, "r") as f:
+            oldIndex = f.readline()
+
+    newIndex = os.getenv("CONSUL_INDEX", "no-index")
+
+    if oldIndex != newIndex:
+        triggers = set()
+        for event in events:
+            if event["Name"].startswith("chant:"):
+                target = event["Name"].removeprefix("chant:")
+                if target not in validTargets:
+                    print(f"Skipping invalid target: {target}")
+                    continue
+                with open(f"/run/chant/{target}", "wb") as f:
+                    if event["Payload"] is not None:
+                        f.write(base64.b64decode(event["Payload"]))
+                triggers.add(target)
+
+        for trigger in triggers:
+            subprocess.run(["${config.systemd.package}/bin/systemctl", "start", f"{trigger}.service"])
+
+    with open(indexFile, "w") as f:
+        f.write(newIndex)
+  '';
+in
+{
+  systemd.services.chant-listener = {
+    description = "Chant Listener";
+    wantedBy = [ "multi-user.target" ];
+    requires = [ "consul-ready.service" ];
+    after = [ "consul-ready.service" ];
+    serviceConfig = {
+      ExecStart = "${config.services.consul.package}/bin/consul watch --type=event ${eventHandler}";
+
+      RuntimeDirectory = "chant";
+      RuntimeDirectoryMode = "0700";
+      CacheDirectory = "chant";
+      CacheDirectoryMode = "0700";
+
+      RestartSec = 60;
+      Restart = "always";
+      IPAddressDeny = [ "any" ];
+      IPAddressAllow = [ consul.ipv4 ];
+    };
+    environment = {
+      CONSUL_HTTP_ADDR = consul.tuple;
+    };
+  };
+}

cluster/services/consul/agent.nix

@@ -1,7 +1,7 @@
-{ config, cluster, lib, tools, ... }:
+{ config, cluster, depot, ... }:
 
 let
-  inherit (tools.meta) domain;
+  inherit (depot.lib.meta) domain;
   inherit (config.networking) hostName;
   inherit (cluster.config) hostLinks;
   cfg = cluster.config.services.consul;
@@ -10,9 +10,12 @@ let
 in
 
 {
+  links.consulAgent.protocol = "http";
+
   services.consul = {
     enable = true;
     webUi = true;
+    package = depot.packages.consul;
     extraConfig = {
       datacenter = "eu-central";
       domain = "sd-magic.${domain}.";
@@ -21,12 +24,16 @@ in
       node_name = config.networking.hostName;
       bind_addr = hl.ipv4;
       ports.serf_lan = hl.port;
-      retry_join = map (hostName: hostLinks.${hostName}.consul.tuple) cfg.otherNodes.agent;
+      retry_join = map (hostName: hostLinks.${hostName}.consul.tuple) (cfg.otherNodes.agent hostName);
+      bootstrap_expect = builtins.length cfg.nodes.agent;
+      addresses.http = config.links.consulAgent.ipv4;
+      ports.http = config.links.consulAgent.port;
     };
   };
 
   services.grafana-agent.settings.integrations.consul_exporter = {
     enabled = true;
     instance = hostName;
+    server = config.links.consulAgent.url;
   };
 }

cluster/services/consul/default.nix

@@ -11,10 +11,23 @@ in
     };
   });
   services.consul = {
-    nodes.agent = [ "checkmate" "thunderskin" "VEGAS" "prophet" ];
-    nixos.agent = [
-      ./agent.nix
-      ./remote-api.nix
-    ];
+    nodes = {
+      agent = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
+      ready = config.services.consul.nodes.agent;
+    };
+    nixos = {
+      agent = [
+        ./agent.nix
+        ./remote-api.nix
+      ];
+      ready = ./ready.nix;
+    };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" ];
+      settings = ./test.nix;
+    };
   };
+
+  dns.records."consul-remote.internal".consulService = "consul-remote";
 }

cluster/services/consul/ready.nix

@@ -0,0 +1,54 @@
+{ config, lib, pkgs, ... }:
+
+let
+  consulReady = pkgs.writers.writeHaskellBin "consul-ready" {
+    libraries = with pkgs.haskellPackages; [ aeson http-conduit watchdog ];
+  } ''
+    {-# LANGUAGE OverloadedStrings #-}
+    import Control.Watchdog
+    import Control.Exception
+    import System.IO
+    import Network.HTTP.Simple
+    import Data.Aeson
+
+    flushLogger :: WatchdogLogger String
+    flushLogger taskErr delay = do
+      defaultLogger taskErr delay
+      hFlush stdout
+
+    data ConsulHealth = ConsulHealth {
+      healthy :: Bool
+    }
+
+    instance FromJSON ConsulHealth where
+      parseJSON (Object v) = ConsulHealth <$> (v .: "Healthy")
+
+    handleException ex = case ex of
+      (SomeException _) -> return $ Left "Consul is not active"
+
+    main :: IO ()
+    main = watchdog $ do
+      setInitialDelay 300_000
+      setMaximumDelay 30_000_000
+      setLoggingAction flushLogger
+      watch $ handle handleException $ do
+          res <- httpJSON "${config.links.consulAgent.url}/v1/operator/autopilot/health"
+          case getResponseBody res of
+            ConsulHealth True -> return $ Right ()
+            ConsulHealth False -> return $ Left "Consul is unhealthy"
+  '';
+in
+
+{
+  systemd.services.consul-ready = {
+    description = "Wait for Consul";
+    requires = lib.mkIf config.services.consul.enable [ "consul.service" ];
+    after = lib.mkIf config.services.consul.enable [ "consul.service" ];
+    serviceConfig = {
+      ExecStart = lib.getExe consulReady;
+      DynamicUser = true;
+      TimeoutStartSec = "5m";
+      Type = "oneshot";
+    };
+  };
+}

cluster/services/consul/remote-api.nix

@@ -1,14 +1,15 @@
-{ config, cluster, depot, lib, tools, ... }:
+{ config, depot, lib, ... }:
 
 let
-  inherit (tools.meta) domain;
-  inherit (depot.reflection) hyprspace;
+  inherit (depot.lib.meta) domain;
   frontendDomain = "consul-remote.internal.${domain}";
+
+  inherit (config.reflection.interfaces.vstub) addr;
 in
 
 {
-  services.nginx.virtualHosts.${frontendDomain} = tools.nginx.vhosts.proxy "http://127.0.0.1:8500" // {
-    listenAddresses = lib.singleton hyprspace.addr;
+  services.nginx.virtualHosts.${frontendDomain} = depot.lib.nginx.vhosts.proxy config.links.consulAgent.url // {
+    listenAddresses = lib.singleton addr;
     enableACME = false;
     useACMEHost = "internal.${domain}";
   };
@@ -18,13 +19,13 @@ in
     mode = "external";
     definition = {
       name = "consul-remote";
-      address = hyprspace.addr;
+      address = addr;
       port = 443;
       checks = [
         {
           name = "Frontend";
           id = "service:consul-remote:frontend";
-          http = "https://${hyprspace.addr}/v1/status/leader";
+          http = "https://${addr}/v1/status/leader";
           tls_server_name = frontendDomain;
           header.Host = lib.singleton frontendDomain;
           interval = "60s";
@@ -32,7 +33,7 @@ in
         {
           name = "Backend";
           id = "service:consul-remote:backend";
-          http = "http://127.0.0.1:8500/v1/status/leader";
+          http = "${config.links.consulAgent.url}/v1/status/leader";
           interval = "30s";
         }
       ];

cluster/services/consul/test.nix

@@ -0,0 +1,24 @@
+{ lib, ... }:
+
+{
+  defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
+
+  testScript = ''
+    import json
+
+    start_all()
+
+    with subtest("should form cluster"):
+      nodes = [ n for n in machines if n != nowhere ]
+      for machine in nodes:
+        machine.succeed("systemctl start consul-ready.service")
+      for machine in nodes:
+        consulConfig = json.loads(machine.succeed("cat /etc/consul.json"))
+        addr = consulConfig["addresses"]["http"]
+        port = consulConfig["ports"]["http"]
+        setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
+        memberList = machine.succeed(f"{setEnv} consul members --status=alive")
+        for machine2 in nodes:
+          assert machine2.name in memberList
+  '';
+}

cluster/services/content-delivery/default.nix

@@ -0,0 +1,7 @@
+{
+  garage = {
+    buckets.content-delivery.web.enable = true;
+  };
+
+  ways.cdn.bucket = "content-delivery";
+}

cluster/services/dns/acme-dns-direct-key.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A 9n5IirzhNBIPRj9Gir+/yQhFH830sgfezsqY5Ulzz3o
+VItDDdgfTFcvSq/QpIqTHnfr1VHqfI6nPz+WWKYQjHw
+-> ssh-ed25519 5/zT0w MfBZrd8wJjoProwdPqsS9CZ9aYNTXgrYviFDwuchQVM
+8WKPYO+i1ZSkPYDrHVJ5Pclj2hEzqwAtf31Agzei444
+-> ssh-ed25519 TCgorQ 3QYtSx/2eiFp54W60F8FlERfHx+DUfnXXfugiXNPECg
+pBx3If3qihD//Aq8hDWCt+U1tiWoCLUDcg/RyVCD0D0
+-> ssh-ed25519 P/nEqQ NImm+vKuL50G2kdD2svmfkwsovmryCSyKyhnZ0duDDo
+U0PTKHiCj4SxomnJdgubo+3sStSE+YwvCnrRl7aAS1Q
+-> ssh-ed25519 FfIUuQ SRgJoBIoW71SiXuHqlnGqRG5AKUrnQy0ecwznGEGTHA
+a0IS3hjMln1tWEjo30A6gYtaV7TJSY4SZDarhahMoLk
+-> ssh-ed25519 d3WGuA 0qVNcrYe53Wo46zFJs6UZtX0dq7TUy72WGdGpLqB3yo
+jTHE9PfhRw5lbBlfznS+ThkSsab3ioearf91xyPBfdQ
+-> ssh-ed25519 YIaSKQ CCcBlAOms2aSkB6pws6tN+4Gf551idI9Zq0rokd0P1c
+/3oFp6hf+jggurbcuu0cXdDL8lr6m/LTHEeNgiJt2gg
+-> K&wn-grease ,Ewz Jc+dQQRp NU~.
+FvDOuTGNaLuCfDelsrRbthjuJT9fBZAQ+kz+7Stoc2wciXV1YpCcOYDHSF38OwRF
+X/pyjVudbJKS0Mphda6phw
+--- 3JFwCzeJsIgRkTpmy9MAvQ64BCZoa98kNKOuT57WI6Y
+&ÀO¿¹¸p ž-ÚP¶.+"<22>ðjÔG«
+ëÇÐs<>gnz[t	
‘ØóÄD÷•RŽÄ½±šmÃl<!Çê6;³Ù÷<C399>†8{	vmvJJ;lR<6C>×[Yà3˜XPËÜ<C38B>ÈPCÿè¯&¦àåYû×2ÃǤxVúÈF{zäQ‹hnW*I$é;°Yc¨@7Ö-k4—À§xãͶx¿µ% RÝ<52>¤$z|»Ê“ñœ¹¯<C2B9>ëñ3

cluster/services/dns/admin.nix

@@ -1,109 +0,0 @@
-{ cluster, config, lib, pkgs, tools, ... }:
-
-let
-  inherit (tools.meta) domain;
-  inherit (config.links) pdnsAdmin;
-  inherit (cluster.config) vars;
-
-  pdns-api = cluster.config.links.powerdns-api;
-
-  dataDirUI = "/srv/storage/private/powerdns-admin";
-
-  translateConfig = withQuotes: cfg: let
-    pythonValue = val: if lib.isString val then "'${val}'"
-      else if lib.isAttrs val && val ? file then "[(f.read().strip('\\n'), f.close()) for f in [open('${val.file}')]][0][0]"
-      else if lib.isAttrs val && val ? env then "__import__('os').getenv('${val.env}')"
-      else if lib.isBool val then (if val then "True" else "False")
-      else if lib.isInt val then toString val
-      else throw "translateConfig: unsupported value type";
-
-    quote = str: if withQuotes then pythonValue str else str;
-
-    configList = lib.mapAttrsToList (n: v: "${n}=${quote v}") cfg;
-  in lib.concatStringsSep "\n" configList;
-
-in {
-  age.secrets = {
-    pdns-admin-oidc-secrets = {
-      file = ./pdns-admin-oidc-secrets.age;
-      mode = "0400";
-    };
-    pdns-admin-salt = {
-      file = ./pdns-admin-salt.age;
-      mode = "0400";
-      owner = "powerdnsadmin";
-      group = "powerdnsadmin";
-    };
-    pdns-admin-secret = {
-      file = ./pdns-admin-secret.age;
-      mode = "0400";
-      owner = "powerdnsadmin";
-      group = "powerdnsadmin";
-    };
-    pdns-api-key = vars.pdns-api-key-secret // { owner = "powerdnsadmin"; };
-  };
-
-  links.pdnsAdmin.protocol = "http";
-
-  networking.firewall = {
-    allowedTCPPorts = [ 53 ];
-    allowedUDPPorts = [ 53 ];
-  };
-
-  systemd.tmpfiles.rules = [
-    "d '${dataDirUI}' 0700 powerdnsadmin powerdnsadmin - -"
-  ];
-
-  services.powerdns = {
-    enable = true;
-    extraConfig = translateConfig false {
-      api = "yes";
-      webserver-allow-from = "127.0.0.1, ${vars.meshNet.cidr}";
-      webserver-address = pdns-api.ipv4;
-      webserver-port = pdns-api.portStr;
-      api-key = "$scrypt$ln=14,p=1,r=8$ZRgztsniH1y+F7P/RkXq/w==$QTil5kbJPzygpeQRI2jgo5vK6fGol9YS/NVR95cmWRs=";
-    };
-  };
-
-  services.powerdns-admin = {
-    enable = true;
-    secretKeyFile = config.age.secrets.pdns-admin-secret.path;
-    saltFile = config.age.secrets.pdns-admin-salt.path;
-    extraArgs = [ "-b" pdnsAdmin.tuple ];
-    config = translateConfig true {
-      SQLALCHEMY_DATABASE_URI = "sqlite:///${dataDirUI}/pda.db";
-      PDNS_VERSION = pkgs.pdns.version;
-      PDNS_API_URL = pdns-api.url;
-      PDNS_API_KEY.file = config.age.secrets.pdns-api-key.path;
-
-      SIGNUP_ENABLED = false;
-      OIDC_OAUTH_ENABLED = true;
-      OIDC_OAUTH_KEY = "net.privatevoid.dnsadmin1";
-      OIDC_OAUTH_SECRET.env = "OIDC_OAUTH_SECRET";
-      OIDC_OAUTH_SCOPE = "openid profile email roles";
-
-      OIDC_OAUTH_METADATA_URL = "https://login.${domain}/auth/realms/master/.well-known/openid-configuration";
-    };
-  };
-
-  systemd.services.powerdns-admin.serviceConfig = {
-    BindPaths = [
-      dataDirUI
-      config.age.secrets.pdns-api-key.path
-    ];
-    TimeoutStartSec = "300s";
-    EnvironmentFile = config.age.secrets.pdns-admin-oidc-secrets.path;
-  };
-
-  services.nginx.virtualHosts."dnsadmin.${domain}" = lib.recursiveUpdate
-  (tools.nginx.vhosts.proxy pdnsAdmin.url)
-  # backend sends really big headers for some reason
-  # increase buffer size accordingly
-  {
-    locations."/".extraConfig = ''
-      proxy_busy_buffers_size 512k;
-      proxy_buffers 4 512k;
-      proxy_buffer_size 256k;
-    '';
-  };
-}

cluster/services/dns/authoritative.nix

@@ -1,30 +1,50 @@
-{ cluster, config, depot, lib, tools, ... }:
+{ cluster, config, depot, lib, pkgs, ... }:
 
 let
-  inherit (depot.reflection) interfaces;
-  inherit (tools.meta) domain;
+  inherit (config.reflection) interfaces;
+  inherit (depot.lib.meta) domain;
   inherit (config.networking) hostName;
 
   link = cluster.config.hostLinks.${hostName}.dnsAuthoritative;
   patroni = cluster.config.links.patroni-pg-access;
+  inherit (cluster.config.hostLinks.${hostName}) acmeDnsApi;
 
-  otherDnsServers = lib.pipe (with cluster.config.services.dns.otherNodes; master ++ slave) [
+  otherDnsServers = lib.pipe (cluster.config.services.dns.otherNodes.authoritative hostName) [
     (map (node: cluster.config.hostLinks.${node}.dnsAuthoritative.tuple))
     (lib.concatStringsSep " ")
   ];
 
-  translateConfig = cfg: let
-    configList = lib.mapAttrsToList (n: v: "${n}=${v}") cfg;
-  in lib.concatStringsSep "\n" configList;
+  recordsList = lib.mapAttrsToList (lib.const lib.id) cluster.config.dns.records;
+  recordsPartitioned = lib.partition (record: record.rewrite.target == null) recordsList;
+
+  staticRecords = let
+    escape = type: {
+      TXT = builtins.toJSON;
+    }.${type} or lib.id;
+
+    recordName = record: {
+      "@" = "${record.root}.";
+    }.${record.name} or "${record.name}.${record.root}.";
+  in lib.flatten (
+    map (record: map (target: "${recordName record} ${record.type} ${escape record.type target}") record.target) recordsPartitioned.right
+  );
+
+  rewrites = map (record: let
+    maybeEscapeRegex = str: if record.rewrite.type == "regex" then "${lib.escapeRegex str}$" else str;
+  in "rewrite stop name ${record.rewrite.type} ${record.name}${maybeEscapeRegex ".${record.root}."} ${record.rewrite.target}. answer auto") recordsPartitioned.wrong;
+
+  rewriteConf = pkgs.writeText "coredns-rewrites.conf" ''
+    rewrite stop type DS DS
+    rewrite stop type NS NS
+    rewrite stop type SOA SOA
+    ${lib.concatStringsSep "\n" rewrites}
+  '';
 in {
   links.localAuthoritativeDNS = {};
 
   age.secrets = {
-    pdns-db-credentials = {
-      file = ./pdns-db-credentials.age;
-      mode = "0400";
-      owner = "pdns";
-      group = "pdns";
+    acmeDnsDirectKey = {
+      file = ./acme-dns-direct-key.age;
     };
   };
 
@@ -33,23 +53,37 @@ in {
     allowedUDPPorts = [ 53 ];
   };
 
-  services.powerdns = {
+  services.acme-dns = {
     enable = true;
-    extraConfig = translateConfig {
-      launch = "gpgsql";
-      local-address = config.links.localAuthoritativeDNS.tuple;
-      gpgsql-host = patroni.ipv4;
-      gpgsql-port = patroni.portStr;
-      gpgsql-dbname = "powerdns";
-      gpgsql-user = "powerdns";
-      gpgsql-extra-connection-parameters = "passfile=${config.age.secrets.pdns-db-credentials.path}";
-      version-string = "Private Void DNS";
-      enable-lua-records = "yes";
-      expand-alias = "yes";
-      resolver = "127.0.0.1:8600";
+    package = depot.packages.acme-dns;
+    settings = {
+      general = {
+        listen = config.links.localAuthoritativeDNS.tuple;
+        inherit domain;
+        nsadmin = "hostmaster.${domain}";
+        nsname = "eu1.ns.${domain}";
+        records = staticRecords;
+      };
+      api = {
+        ip = acmeDnsApi.ipv4;
+        inherit (acmeDnsApi) port;
+      };
+      database = {
+        engine = "postgres";
+        connection = "postgres://acmedns@${patroni.tuple}/acmedns?sslmode=disable";
+      };
     };
   };
 
+  services.locksmith.waitForSecrets.acme-dns = [
+    "patroni-acmedns"
+  ];
+
+  systemd.services.acme-dns.serviceConfig.EnvironmentFile = with config.age.secrets; [
+    "/run/locksmith/patroni-acmedns"
+    acmeDnsDirectKey.path
+  ];
+
   services.coredns = {
     enable = true;
     config = ''
@@ -60,10 +94,14 @@ in {
           success 4000 86400
           denial 0
           prefetch 3
-          serve_stale 86400s
+          serve_stale 86400s verify
+        }
+        template ANY DS {
+          rcode NXDOMAIN
         }
         forward service.eu-central.sd-magic.${domain} 127.0.0.1:8600
         forward addr.eu-central.sd-magic.${domain} 127.0.0.1:8600
+        import ${rewriteConf}
         forward . ${config.links.localAuthoritativeDNS.tuple} ${otherDnsServers} {
           policy sequential
         }
@@ -72,18 +110,34 @@ in {
   };
 
   systemd.services.coredns = {
-    after = [ "pdns.service" ];
+    after = [ "acme-dns.service" ];
+    serviceConfig = {
+      MemoryMax = "200M";
+      MemorySwapMax = "50M";
+      CPUQuota = "25%";
+    };
   };
 
-  consul.services.pdns = {
-    mode = "external";
-    definition = {
-      name = "authoritative-dns-backend";
-      address = config.links.localAuthoritativeDNS.ipv4;
-      port = config.links.localAuthoritativeDNS.port;
+  consul.services = {
+    authoritative-dns = {
+      unit = "acme-dns";
+      definition = {
+        name = "authoritative-dns-backend";
+        address = config.links.localAuthoritativeDNS.ipv4;
+        port = config.links.localAuthoritativeDNS.port;
+        checks = lib.singleton {
+          interval = "60s";
+          tcp = config.links.localAuthoritativeDNS.tuple;
+        };
+      };
+    };
+    acme-dns.definition = {
+      name = "acme-dns";
+      address = acmeDnsApi.ipv4;
+      port = acmeDnsApi.port;
       checks = lib.singleton {
         interval = "60s";
-        tcp = config.links.localAuthoritativeDNS.tuple;
+        http = "${acmeDnsApi.url}/health";
       };
     };
   };

cluster/services/dns/coredns.nix

@@ -1,23 +1,21 @@
-{ cluster, config, depot, lib, pkgs, tools, ... }:
+{ cluster, config, depot, lib, ... }:
 
 let
-  inherit (depot.reflection) interfaces hyprspace;
-  inherit (tools.meta) domain;
-  inherit (config.links) localRecursor;
+  inherit (config.reflection) interfaces;
+  inherit (depot.lib.meta) domain;
   inherit (config.networking) hostName;
 
   link = cluster.config.hostLinks.${hostName}.dnsResolver;
   backend = cluster.config.hostLinks.${hostName}.dnsResolverBackend;
 
-  otherRecursors = lib.pipe (cluster.config.services.dns.otherNodes.coredns) [
+  otherRecursors = lib.pipe (cluster.config.services.dns.otherNodes.coredns hostName) [
     (map (node: cluster.config.hostLinks.${node}.dnsResolverBackend.tuple))
     (lib.concatStringsSep " ")
   ];
 
-  authoritativeServers = lib.pipe (with cluster.config.services.dns.nodes; master ++ slave) [
-    (map (node: cluster.config.hostLinks.${node}.dnsAuthoritative.tuple))
-    (lib.concatStringsSep ";")
-  ];
+  authoritativeServers = map
+    (node: cluster.config.hostLinks.${node}.dnsAuthoritative.tuple)
+    cluster.config.services.dns.nodes.authoritative;
 
   inherit (depot.packages) stevenblack-hosts;
   dot = config.security.acme.certs."securedns.${domain}";
@@ -37,14 +35,17 @@ in
     ];
     before = [ "acme-securedns.${domain}.service" ];
     wants = [ "acme-finished-securedns.${domain}.target" ];
-    serviceConfig.LoadCredential = [
-      "dot-cert.pem:${dot.directory}/fullchain.pem"
-      "dot-key.pem:${dot.directory}/key.pem"
-    ];
+    serviceConfig = {
+      LoadCredential = [
+        "dot-cert.pem:${dot.directory}/fullchain.pem"
+        "dot-key.pem:${dot.directory}/key.pem"
+      ];
+      ExecReload = lib.mkForce [];
+    };
   };
 
   security.acme.certs."securedns.${domain}" = {
-    dnsProvider = "pdns";
+    dnsProvider = "exec";
     # using a different ACME provider because Android Private DNS is fucky
     server = "https://api.buypass.com/acme/directory";
     reloadServices = [
@@ -55,29 +56,29 @@ in
   services.coredns = {
     enable = true;
     config = ''
-      .:${link.portStr} {
-        ${lib.optionalString (interfaces ? vstub) "bind ${interfaces.vstub.addr}"}
-        bind 127.0.0.1
-        bind ${link.ipv4}
-        ${lib.optionalString hyprspace.enable "bind ${hyprspace.addr}"}
+      (localresolver) {
         hosts ${stevenblack-hosts} {
           fallthrough
         }
         chaos "Private Void DNS" info@privatevoid.net
+        forward hyprspace. 127.43.104.80:11355
+        forward ${domain}. ${lib.concatStringsSep " " authoritativeServers} {
+          policy random
+        }
         forward . ${backend.tuple} ${otherRecursors} {
           policy sequential
         }
       }
+      .:${link.portStr} {
+        ${lib.optionalString (interfaces ? vstub) "bind ${interfaces.vstub.addr}"}
+        bind 127.0.0.1
+        bind ${link.ipv4}
+        import localresolver
+      }
       tls://.:853 {
         bind ${interfaces.primary.addr}
         tls {$CREDENTIALS_DIRECTORY}/dot-cert.pem {$CREDENTIALS_DIRECTORY}/dot-key.pem
-        hosts ${stevenblack-hosts} {
-          fallthrough
-        }
-        chaos "Private Void DNS" info@privatevoid.net
-        forward . ${backend.tuple} ${otherRecursors} {
-          policy sequential
-        }
+        import localresolver
       }
     '';
   };
@@ -87,7 +88,7 @@ in
     dnssecValidation = "process";
     forwardZones = {
       # optimize queries against our own domain
-      "${domain}" = authoritativeServers;
+      "${domain}" = lib.concatStringsSep ";" authoritativeServers;
     };
     dns = {
       inherit (backend) port;

cluster/services/dns/default.nix

@@ -1,30 +1,37 @@
 { config, depot, lib, ... }:
 
 let
-  inherit (depot.config) hours;
+  inherit (depot) hours;
   cfg = config.services.dns;
 in
 {
-  vars.pdns-api-key-secret = {
-    file = ./pdns-api-key.age;
-    mode = "0400";
-  };
+  imports = [
+    ./options.nix
+    ./nodes.nix
+    ./ns-records.nix
+  ];
+
   links = {
     dnsResolver = {
       ipv4 = hours.VEGAS.interfaces.vstub.addr;
       port = 53;
     };
-    powerdns-api = {
-      ipv4 = config.vars.mesh.VEGAS.meshIp;
+    acmeDnsApi = {
+      hostname = "acme-dns-challenge.internal.${depot.lib.meta.domain}";
       protocol = "http";
     };
   };
   hostLinks = lib.mkMerge [
-    (lib.genAttrs (with cfg.nodes; master ++ slave) (node: {
+    (lib.genAttrs cfg.nodes.authoritative (node: {
       dnsAuthoritative = {
         ipv4 = hours.${node}.interfaces.primary.addrPublic;
         port = 53;
       };
+      acmeDnsApi = {
+        ipv4 = config.vars.mesh.${node}.meshIp;
+        inherit (config.links.acmeDnsApi) port;
+        protocol = "http";
+      };
     }))
     (lib.genAttrs cfg.nodes.coredns (node: {
       dnsResolver = {
@@ -40,19 +47,34 @@ in
   ];
   services.dns = {
     nodes = {
-      master = [ "VEGAS" ];
-      slave = [ "checkmate" "prophet" ];
+      authoritative = [ "VEGAS" "checkmate" "prophet" ];
       coredns = [ "checkmate" "VEGAS" ];
-      client = [ "checkmate" "thunderskin" "VEGAS" "prophet" ];
+      client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
     };
     nixos = {
-      master = [
-        ./authoritative.nix
-        ./admin.nix
-      ];
-      slave = ./authoritative.nix;
+      authoritative = ./authoritative.nix;
       coredns = ./coredns.nix;
       client = ./client.nix;
     };
+    simulacrum = {
+      enable = true;
+      deps = [ "consul" "acme-client" "patroni" ];
+      settings = ./test.nix;
+    };
+  };
+
+  patroni = {
+    databases.acmedns = {};
+    users.acmedns = {
+      locksmith = {
+        nodes = config.services.dns.nodes.authoritative;
+        format = "envFile";
+      };
+    };
+  };
+
+  dns.records = {
+    securedns.consulService = "securedns";
+    "acme-dns-challenge.internal".consulService = "acme-dns";
   };
 }

cluster/services/dns/nodes.nix

@@ -0,0 +1,11 @@
+{ depot, lib, ... }:
+
+{
+  dns.records = lib.mapAttrs' (name: hour: {
+    name = lib.toLower "${name}.${hour.enterprise.subdomain}";
+    value = {
+      type = "A";
+      target = [ hour.interfaces.primary.addrPublic ];
+    };
+  }) depot.gods.fromLight;
+}

cluster/services/dns/ns-records.nix

@@ -0,0 +1,26 @@
+{ config, depot, lib, ... }:
+
+let
+  cfg = config.services.dns;
+
+  nsNodes = lib.imap1 (idx: node: {
+    name = "eu${toString idx}.ns";
+    value = {
+      type = "A";
+      target = [ depot.hours.${node}.interfaces.primary.addrPublic ];
+    };
+  }) cfg.nodes.authoritative;
+in
+
+{
+  dns.records = lib.mkMerge [
+    (lib.listToAttrs nsNodes)
+    {
+      NS = {
+        name = "@";
+        type = "NS";
+        target = map (ns: "${ns.name}.${depot.lib.meta.domain}.") nsNodes;
+      };
+    }
+  ];
+}

cluster/services/dns/options.nix

@@ -0,0 +1,61 @@
+{ depot, lib, ... }:
+
+with lib;
+
+let
+  recordType = types.submodule ({ config, name, ... }: {
+    options = {
+      root = mkOption {
+        type = types.str;
+        default = depot.lib.meta.domain;
+      };
+      consulServicesRoot = mkOption {
+        type = types.str;
+        default = "service.eu-central.sd-magic.${depot.lib.meta.domain}";
+      };
+      name = mkOption {
+        type = types.str;
+        default = name;
+      };
+
+      type = mkOption {
+        type = types.enum [ "A" "CNAME" "AAAA" "NS" "MX" "SOA" "TXT" ];
+        default = "A";
+      };
+      target = mkOption {
+        type = with types; listOf str;
+      };
+      ttl = mkOption {
+        type = types.ints.unsigned;
+        default = 86400;
+      };
+
+      consulService = mkOption {
+        type = with types; nullOr str;
+        default = null;
+      };
+      rewrite = {
+        target = mkOption {
+          type = with types; nullOr str;
+          default = null;
+        };
+        type = mkOption {
+          type = types.enum [ "exact" "substring" "prefix" "suffix" "regex" ];
+          default = "exact";
+        };
+      };
+    };
+    config = {
+      rewrite.target = mkIf (config.consulService != null) "${config.consulService}.${config.consulServicesRoot}";
+    };
+  });
+in
+
+{
+  options.dns = {
+    records = mkOption {
+      type = with types; attrsOf recordType;
+      default = {};
+    };
+  };
+}

cluster/services/dns/pdns-admin-salt.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A d/YNanH/cHoFLPp8WcCXHh/LQLRwaUa95JiRLbgb8RI
-UPEHpnHHTU6dGKi2MbApEspcpt1lFtFZ4XJjShL7OoE
--> ssh-ed25519 5/zT0w Rv9ZS5P2Eca3npPLR7yym/XTRSDfVmgRwH1pAGR79T8
-4A/KXc2wxxokfDAwWYf0ZTUEzQ8ldkC+zRNZY3KjBTs
--> ssh-ed25519 d3WGuA 2R0kaVjuhU3wT9pjj214zkEaHYNSlMxf9Z+MfBssHwY
-EU5LWk6xfohWM/3sAqYtUvFmRgIPxOLXHnlqbsQ3+ok
--> -|(-grease W=cc~ O2q5
-FZzh/ZwDS2EqvVZ9NErmUwCMN72op1Qy
---- Ducan3ugRJC3dmWLr7+FKok+WmInOgOzW0ccYeqAFAQ
-Ì•ãÆ*Q. SC<53>ûf¹‰*`5<>„ÑÖw"~ÍxwÜ*–ã\‹êÙ"²ÅtŒ	'’É0ï™<C3AF>L£ï

cluster/services/dns/pdns-admin-secret.age

@@ -1,12 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A hUR+UdHnpazhANM8DKToI5Th3lv1aAuxZ1IQKvCOv34
-PvsiSym8YdleDULLnWuTs1x08KO3EmAg/AAjulgrgqE
--> ssh-ed25519 5/zT0w qMXS2xLOLv/+l6brG11i+3FwHdrhlmxZBNtBiU9hu2g
-BlFYPvH4mFJRMHTlHwnBdJb6QcugylwZuT5bgSKcQa0
--> ssh-ed25519 d3WGuA k2fRQ3+HyZP+bb/gkVKQqUmbITJLPm9tGp67DbRfiCs
-RX9CACfYpYKvSqyfXjvEokTGsp4+ECQBD8i1ehD5xRg
--> IB@F$9G-grease
-cXRgUVdIPGEjft1CJA
---- si16Det/GwF7GLHLt0ha8v4rFFeJXyhEylIiqzZVAK8
-Ö°å¤pÐǺ#ê4^©—
~u
Uuç­aòQ´™Bâj˜(N)qÃ<"¤%ì’,V9û5ZÔh§#W«[»ò¶”"Mÿ&”îäøÖýá+%Œ«„SQ€B÷Þ›ÕÀèÕyàÜî<aéó]P‚$´Ä±B¨½qQÑÉQ‡M‰TËt°
-·s¹mÿ~qW–Ö«çêõÜ×Ì=.Q“"ù”–Þø¶ÏnqRk<52>=ÏcÿçüßÃqv¢¾>#ŠÏ«²tïwq,÷	»3YyIq}Ê“ì>sgíz™ûs±Þ ¸ƆFÄPê|ÍüÅ¡=ùÃþ~KQR,DZuÐ+ÕºZGHëa=‹©;ÀõC.ÏuVShÅ$Và€AË9Ð=
?•¢