cluster/services/ways: add simulacrum deps

cluster/services/acme-client/augment.nix

@@ -0,0 +1,60 @@
+{ config, pkgs, ... }:
+
+let
+  lift = config;
+in
+
+{
+  nowhere.names = {
+    "acme-v02.api.letsencrypt.org" = "stepCa";
+    "api.buypass.com" = "stepCa";
+  };
+
+  nodes.nowhere = { config, ... }: {
+    links.stepCa.protocol = "https";
+
+    environment.etc.step-ca-password.text = "";
+
+    services = {
+      step-ca = {
+        enable = true;
+        address = config.links.stepCa.ipv4;
+        inherit (config.links.stepCa) port;
+        intermediatePasswordFile = "/etc/step-ca-password";
+        settings = {
+          root = "${lift.nowhere.certs.ca}/ca.pem";
+          crt = "${lift.nowhere.certs.intermediate}/cert.pem";
+          key = "${lift.nowhere.certs.intermediate}/cert-key.pem";
+          address = config.links.stepCa.tuple;
+          db = {
+            type = "badgerv2";
+            dataSource = "/var/lib/step-ca/db";
+          };
+          authority.provisioners = [
+            {
+              type = "ACME";
+              name = "snakeoil";
+              challenges = [
+                "dns-01"
+                "http-01"
+              ];
+            }
+          ];
+        };
+      };
+
+      nginx.virtualHosts = {
+        "acme-v02.api.letsencrypt.org".locations."/".extraConfig = ''
+          rewrite /directory /acme/snakeoil/directory break;
+        '';
+        "api.buypass.com".locations."/".extraConfig = ''
+          rewrite /acme/directory /acme/snakeoil/directory break;
+        '';
+      };
+    };
+  };
+
+  defaults.environment.etc."dummy-secrets/acmeDnsApiKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
+  defaults.environment.etc."dummy-secrets/acmeDnsDirectKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
+  defaults.environment.etc."dummy-secrets/acmeDnsDbCredentials".text = "PGPASSWORD=simulacrum";
+}

cluster/services/acme-client/client.nix

@@ -32,7 +32,10 @@ in
     owner = "acme";
   };
 
+  security.acme.acceptTerms = true;
+  security.acme.maxConcurrentRenewals = 0;
   security.acme.defaults = {
+    email = depot.lib.meta.adminEmail;
     extraLegoFlags = lib.flatten [
       (map (x: [ "--dns.resolvers" x ]) authoritativeServers)
       "--dns-timeout" "30"
@@ -42,4 +45,38 @@ in
       EXEC_ENV_FILE=${config.age.secrets.acmeDnsApiKey.path}
     '';
   };
+
+  systemd.services = lib.mapAttrs' (name: value: {
+    name = "acme-${name}";
+    value = {
+      distributed.enable = value.dnsProvider != null;
+      preStart = let
+        serverList = lib.pipe authoritativeServers [
+          (map (x: "@${x}"))
+          (map (lib.replaceStrings [":53"] [""]))
+          lib.escapeShellArgs
+        ];
+        domainList = lib.pipe ([ value.domain ] ++ value.extraDomainNames) [
+          (map (x: "${x}."))
+          (map (lib.replaceStrings ["*"] ["x"]))
+          lib.unique
+          lib.escapeShellArgs
+        ];
+      in ''
+        echo Testing availability of authoritative DNS servers
+        for i in {1..60}; do
+          ${pkgs.dig}/bin/dig +short ${serverList} ${domainList} >/dev/null && break
+          echo Retry [$i/60]
+          sleep 10
+        done
+        echo Available
+      '';
+      serviceConfig = {
+        Restart = "on-failure";
+        RestartMaxDelaySec = 30;
+        RestartStesp = 5;
+        RestartMode = "direct";
+      };
+    };
+  }) config.security.acme.certs;
 }

cluster/services/acme-client/default.nix

@@ -2,5 +2,6 @@
   services.acme-client = {
     nodes.client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
     nixos.client = ./client.nix;
+    simulacrum.augments = ./augment.nix;
   };
 }

cluster/services/chant/default.nix

@@ -6,5 +6,6 @@
     nixos.listener = [
       ./listener.nix
     ];
+    simulacrum.deps = [ "consul" ];
   };
 }

cluster/services/dns/authoritative.nix

@@ -43,9 +43,6 @@ in {
   links.localAuthoritativeDNS = {};
 
   age.secrets = {
-    acmeDnsDbCredentials = {
-      file = ./acme-dns-db-credentials.age;
-    };
     acmeDnsDirectKey = {
       file = ./acme-dns-direct-key.age;
     };
@@ -78,8 +75,12 @@ in {
     };
   };
 
+  services.locksmith.waitForSecrets.acme-dns = [
+    "patroni-acmedns"
+  ];
+
   systemd.services.acme-dns.serviceConfig.EnvironmentFile = with config.age.secrets; [
-    acmeDnsDbCredentials.path
+    "/run/locksmith/patroni-acmedns"
     acmeDnsDirectKey.path
   ];
 

cluster/services/dns/default.nix

@@ -58,6 +58,16 @@ in
     };
   };
 
+  patroni = {
+    databases.acmedns = {};
+    users.acmedns = {
+      locksmith = {
+        nodes = config.services.dns.nodes.authoritative;
+        format = "envFile";
+      };
+    };
+  };
+
   dns.records = {
     securedns.consulService = "securedns";
     "acme-dns-challenge.internal".consulService = "acme-dns";

cluster/services/forge/default.nix

@@ -17,9 +17,11 @@
     };
   };
 
-  ways.forge.target = let
+  ways.forge = let
     host = builtins.head config.services.forge.nodes.server;
-  in config.lib.forService "forge" config.hostLinks.${host}.forge.url;
+  in config.lib.forService "forge" {
+    target = config.hostLinks.${host}.forge.url;
+  };
 
   garage = config.lib.forService "forge" {
     keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;

cluster/services/incandescence/default.nix

@@ -0,0 +1,20 @@
+{ config, ... }:
+
+{
+  imports = [
+    ./options.nix
+  ];
+
+  services.incandescence = {
+    nodes = {
+      provider = config.services.consul.nodes.agent;
+    };
+    nixos = {
+      provider = [
+        ./provider.nix
+        ./provider-options.nix
+      ];
+    };
+    simulacrum.deps = [ "consul" ];
+  };
+}

cluster/services/incandescence/options.nix

@@ -0,0 +1,22 @@
+{ lib, ... }:
+
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) attrsOf listOf submodule str;
+in
+
+{
+  options.incandescence = {
+    providers = mkOption {
+      type = attrsOf (submodule ({ name, ... }: {
+        options = {
+          objects = mkOption {
+            type = attrsOf (listOf str);
+            default = { };
+          };
+        };
+      }));
+      default = { };
+    };
+  };
+}

cluster/services/incandescence/provider-options.nix

@@ -0,0 +1,72 @@
+{ lib, ... }:
+
+let
+  inherit (lib) mkEnableOption mkOption;
+  inherit (lib.types) attrsOf functionTo ints listOf nullOr package submodule str;
+in
+
+{
+  options.services.incandescence = {
+    providers = mkOption {
+      type = attrsOf (submodule ({ name, ... }: {
+        options = {
+          locksmith = mkEnableOption "Locksmith integration";
+
+          wantedBy = mkOption {
+            type = listOf str;
+          };
+
+          partOf = mkOption {
+            type = listOf str;
+          };
+
+          wants = mkOption {
+            type = listOf str;
+            default = [ ];
+          };
+
+          after = mkOption {
+            type = listOf str;
+            default = [ ];
+          };
+
+          packages = mkOption {
+            type = listOf package;
+            default = [ ];
+          };
+
+          formulae = mkOption {
+            type = attrsOf (submodule ({ ... }: {
+              options = {
+                deps = mkOption {
+                  type = listOf str;
+                  default = [ ];
+                };
+
+                create = mkOption {
+                  type = functionTo str;
+                };
+
+                change = mkOption {
+                  type = nullOr (functionTo str);
+                  default = null;
+                };
+
+                destroy = mkOption {
+                  type = str;
+                };
+
+                destroyAfterDays = mkOption {
+                  type = ints.unsigned;
+                  default = 0;
+                };
+              };
+            }));
+            default = { };
+          };
+        };
+      }));
+      default = { };
+    };
+  };
+}

cluster/services/incandescence/provider.nix

@@ -0,0 +1,128 @@
+{ cluster, config, lib, ... }:
+
+let
+  inherit (lib) concatStringsSep escapeShellArg flatten filter filterAttrs length mapAttrs mapAttrs' mapAttrsToList mkIf mkMerge pipe stringToCharacters;
+
+  cfg = config.services.incandescence;
+  clusterCfg = cluster.config.incandescence;
+in
+
+{
+  systemd.services = pipe cfg.providers [
+    (mapAttrsToList (provider: providerConfig: pipe providerConfig.formulae [
+      (mapAttrsToList (formula: formulaConfig: let
+        kvRoot = "services/incandescence/providers/${provider}/formulae/${formula}";
+        time = "$(date +%s)";
+      in {
+        "ignite-${provider}-${formula}-create" = {
+          description = "Ignite Creation: ${provider} - ${formula}";
+          wantedBy = [ "incandescence-${provider}.target" ];
+          before = [ "incandescence-${provider}.target" ];
+          wants = providerConfig.wants ++ map (dep: "ignite-${provider}-${dep}-create.service") formulaConfig.deps;
+          after = providerConfig.after ++ map (dep: "ignite-${provider}-${dep}-create.service") formulaConfig.deps;
+          serviceConfig.Type = "oneshot";
+          distributed.enable = true;
+          path = [ config.services.consul.package ] ++ providerConfig.packages;
+          script = pipe clusterCfg.providers.${provider}.objects.${formula} [
+            (map (object: ''
+              if ! consul kv get ${kvRoot}/${object}/alive >/dev/null; then
+                echo "Create ${formula}: ${object}"
+                if (
+                ${formulaConfig.create object}
+                )
+                then
+                  consul kv put ${kvRoot}/${object}/alive true
+                  consul kv delete ${kvRoot}/${object}/destroyOn
+                else
+                  echo "Creation failed: ${object}"
+                fi
+              fi
+            ''))
+            (concatStringsSep "\n")
+          ];
+        };
+        "ignite-${provider}-${formula}-change" = mkIf (formulaConfig.change != null) {
+          description = "Ignite Change: ${provider} - ${formula}";
+          wantedBy = [ "incandescence-${provider}.target" ];
+          before = [ "incandescence-${provider}.target" ];
+          wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-create.service" ] ++ map (dep: "ignite-${provider}-${dep}-change.service") formulaConfig.deps;
+          after = providerConfig.after ++ [ "ignite-${provider}-${formula}-create.service" ] ++ map (dep: "ignite-${provider}-${dep}-change.service") formulaConfig.deps;
+          serviceConfig.Type = "oneshot";
+          distributed.enable = true;
+          path = [ config.services.consul.package ] ++ providerConfig.packages;
+          script = pipe clusterCfg.providers.${provider}.objects.${formula} [
+            (map (object: ''
+              echo "Change ${formula}: ${object}"
+              (
+              ${formulaConfig.change object}
+              ) || echo "Change failed: ${object}"
+            ''))
+            (concatStringsSep "\n")
+          ];
+        };
+        "ignite-${provider}-${formula}-destroy" = {
+          description = "Ignite Destruction: ${provider} - ${formula}";
+          wantedBy = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
+          before = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
+          wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-change.service" ];
+          after = providerConfig.after ++ [ "ignite-${provider}-${formula}-change.service" ];
+          serviceConfig.Type = "oneshot";
+          distributed.enable = true;
+          path = [ config.services.consul.package ] ++ providerConfig.packages;
+          script = let
+            fieldNum = pipe kvRoot [
+              stringToCharacters
+              (filter (x: x == "/"))
+              length
+              (builtins.add 2)
+              toString
+            ];
+            keyFilter = pipe clusterCfg.providers.${provider}.objects.${formula} [
+              (map (x: escapeShellArg "^${x}$"))
+              (concatStringsSep " \\\n  -e ")
+            ];
+            destroyAfterDays = toString formulaConfig.destroyAfterDays;
+          in ''
+            consul kv get --keys ${kvRoot}/ | cut -d/ -f${fieldNum} | grep -v -e ${keyFilter} | while read object; do
+              if consul kv get ${kvRoot}/$object/alive >/dev/null; then
+                destroyOn="$(consul kv get ${kvRoot}/$object/destroyOn || true)"
+                if [[ -z "$destroyOn" && "${destroyAfterDays}" -ne 0 ]]; then
+                  echo "Schedule ${formula} for destruction in ${destroyAfterDays} days: $object"
+                  consul kv put ${kvRoot}/$object/destroyOn "$((${time} + 86400 * ${destroyAfterDays}))"
+                elif [[ "${destroyAfterDays}" -eq 0 || "${time}" -ge "$destroyOn" ]]; then
+                  echo "Destroy ${formula}: $object"
+                  export OBJECT="$object"
+                  if (
+                  ${formulaConfig.destroy}
+                  )
+                  then
+                    consul kv delete --recurse ${kvRoot}/$object
+                  else
+                    echo "Destruction failed: $object"
+                  fi
+                else
+                  echo "Scheduled for destruction on $destroyOn (now: ${time})"
+                fi
+              fi
+            done
+          '';
+        };
+      }))
+    ]))
+    flatten
+    mkMerge
+  ];
+
+  systemd.targets = mapAttrs' (provider: providerConfig: {
+    name = "incandescence-${provider}";
+    value = {
+      description = "An Incandescence | ${provider}";
+      inherit (providerConfig) wantedBy partOf;
+    };
+  }) cfg.providers;
+
+  services.locksmith.providers = mapAttrs (provider: providerConfig: {
+    wantedBy = [ "incandescence-${provider}.target" ];
+    after = [ "incandescence-${provider}.target" ];
+  }) (filterAttrs (_: providerConfig: providerConfig.locksmith) cfg.providers);
+}

cluster/services/locksmith/default.nix

@@ -14,5 +14,6 @@
         ./provider.nix
       ];
     };
+    simulacrum.deps = [ "chant" "consul" ];
   };
 }

cluster/services/locksmith/provider.nix

@@ -28,6 +28,10 @@ in
                 command = mkOption {
                   type = types.coercedTo types.package (package: "${package}") types.str;
                 };
+                checkUpdate = mkOption {
+                  type = types.coercedTo types.package (package: "${package}") types.str;
+                  default = "true";
+                };
                 owner = mkOption {
                   type = types.str;
                   default = "root";
@@ -72,20 +76,27 @@ in
         activeNodes = lib.unique (lib.flatten (lib.mapAttrsToList (_: secret: secret.nodes) activeSecrets));
         secretNames = map (name: "${providerRoot}-${name}/") (lib.attrNames activeSecrets);
 
-        createSecret = { path, nodes, owner, mode, group, command }: ''
-          consul kv put ${lib.escapeShellArg path}/mode ${lib.escapeShellArg mode}
-          consul kv put ${lib.escapeShellArg path}/owner ${lib.escapeShellArg owner}
-          consul kv put ${lib.escapeShellArg path}/group ${lib.escapeShellArg group}
-          ${lib.concatStringsSep "\n" (map (node: ''
-            consul kv put ${lib.escapeShellArg path}/recipient/${node} "$( (${command}) | age --encrypt --armor -r ${lib.escapeShellArg depot.hours.${node}.ssh.id.publicKey})"
-          '') nodes)}
+        createSecret = { path, nodes, owner, mode, group, command, checkUpdate }: ''
+          if (${checkUpdate}); then
+            consul kv put ${lib.escapeShellArg path}/mode ${lib.escapeShellArg mode}
+            consul kv put ${lib.escapeShellArg path}/owner ${lib.escapeShellArg owner}
+            consul kv put ${lib.escapeShellArg path}/group ${lib.escapeShellArg group}
+            secret="$(mktemp -ut)"
+            (${command}) > "$secret"
+            ${lib.concatStringsSep "\n" (map (node: ''
+              consul kv put ${lib.escapeShellArg path}/recipient/${node} "$(age < "$secret" --encrypt --armor -r ${lib.escapeShellArg depot.hours.${node}.ssh.id.publicKey})"
+            '') nodes)}
+          else
+            echo Skipping update for ${lib.escapeShellArg path}
+          fi
         '';
       in ''
         # create/update secrets
+        umask 77
         ${lib.pipe activeSecrets [
           (lib.mapAttrsToList (secretName: secretConfig: createSecret {
             path = "${providerRoot}-${secretName}";
-            inherit (secretConfig) nodes mode owner group command;
+            inherit (secretConfig) nodes mode owner group command checkUpdate;
           }))
           (lib.concatStringsSep "\n")
         ]}

cluster/services/nginx/nginx.nix

@@ -1,10 +1,6 @@
-{ config, depot, ... }:
+{ config, ... }:
 
-let
-  inherit (depot.lib.meta) adminEmail;
-in {
-  security.acme.defaults.email = adminEmail;
-  security.acme.acceptTerms = true;
+{
   services.nginx = {
     enable = true;
     recommendedProxySettings = true;

cluster/services/patroni/create-databases.nix

@@ -0,0 +1,87 @@
+{ cluster, config, lib, pkgs, ... }:
+
+let
+  inherit (cluster.config.services.patroni) secrets;
+
+  patroni = cluster.config.links.patroni-pg-access;
+
+  cfg = cluster.config.patroni;
+
+  writeQueryFile = pkgs.writeText "patroni-query.sql";
+
+  psqlRunFile = file: ''
+    export PGPASSWORD="$(< ${secrets.PATRONI_SUPERUSER_PASSWORD.path})"
+    while ! ${config.services.patroni.postgresqlPackage}/bin/psql 'host=${patroni.ipv4} port=${patroni.portStr} dbname=postgres user=postgres' --tuples-only --csv --file="${file}"; do
+      sleep 3
+    done
+  '';
+
+  psql = query: psqlRunFile (writeQueryFile query);
+
+  psqlSecret = getSecret: queryTemplate: let
+    queryTemplateFile = writeQueryFile queryTemplate;
+  in ''
+    umask 77
+    secretFile="$(mktemp -ut patroniSecret.XXXXXXXXXXXXXXXX)"
+    queryFile="$(mktemp -ut patroniQuery.XXXXXXXXXXXXXXXX)"
+    trap "rm -f $secretFile $queryFile" EXIT
+    ${getSecret} > "$secretFile"
+    cp --no-preserve=mode ${queryTemplateFile} "$queryFile"
+    ${pkgs.replace-secret}/bin/replace-secret '@SECRET@' "$secretFile" "$queryFile"
+    ${psqlRunFile "$queryFile"}
+  '';
+
+  genPassword = pkgs.writeShellScript "patroni-generate-user-password" ''
+    umask 77
+    base64 -w0 /dev/urandom | tr -d /+ | head -c256 | tee "/run/keys/locksmith-provider-patroni-$1"
+  '';
+in
+
+{
+  services.incandescence.providers.patroni = lib.mkIf config.services.haproxy.enable {
+    locksmith = true;
+    wantedBy = [ "patroni.service" "multi-user.target" ];
+    partOf = [ "patroni.service" ];
+    wants = [ "postgresql.service" ];
+    after = [ "postgresql.service" ];
+
+    formulae = {
+      user = {
+        destroyAfterDays = 0;
+        create = user: psqlSecret "${genPassword} ${user}" ''
+          CREATE USER ${user} PASSWORD '@SECRET@';
+        '';
+        destroy = psqlSecret "printenv OBJECT" ''
+          DROP USER @SECRET@;
+        '';
+      };
+      database = {
+        destroyAfterDays = 30;
+        deps = [ "user" ];
+        create = db: psql ''
+          CREATE DATABASE ${db} OWNER ${cfg.databases.${db}.owner};
+        '';
+        destroy = psqlSecret "printenv OBJECT" ''
+          DROP DATABASE @SECRET@;
+        '';
+      };
+    };
+  };
+
+  services.locksmith.providers.patroni = lib.mkIf config.services.haproxy.enable {
+    secrets = lib.mapAttrs (user: userConfig: {
+      command = {
+        envFile = ''
+          echo "PGPASSWORD=$(cat /run/keys/locksmith-provider-patroni-${user})"
+          rm -f /run/keys/locksmith-provider-patroni-${user}
+        '';
+        pgpass = ''
+          echo "*:*:*:${user}:$(cat /run/keys/locksmith-provider-patroni-${user})"
+          rm -f /run/keys/locksmith-provider-patroni-${user}
+        '';
+      }.${userConfig.locksmith.format};
+      checkUpdate = "test -e /run/keys/locksmith-provider-patroni-${user}";
+      inherit (userConfig.locksmith) nodes;
+    }) cfg.users;
+  };
+}

cluster/services/patroni/default.nix

@@ -1,6 +1,11 @@
-{ config, lib, ... }:
+{ config, ... }:
 
 {
+  imports = [
+    ./options.nix
+    ./incandescence.nix
+  ];
+
   links = {
     patroni-pg-internal.ipv4 = "0.0.0.0";
     patroni-api.ipv4 = "0.0.0.0";
@@ -15,6 +20,7 @@
       worker = [
         ./worker.nix
         ./metrics.nix
+        ./create-databases.nix
       ];
       haproxy = ./haproxy.nix;
     };
@@ -30,5 +36,6 @@
       PATRONI_REWIND_PASSWORD = default;
       metricsCredentials.nodes = nodes.worker;
     };
+    simulacrum.deps = [ "consul" "incandescence" "locksmith" ];
   };
 }

cluster/services/patroni/incandescence.nix

@@ -0,0 +1,10 @@
+{ config, lib, ... }:
+
+{
+  incandescence.providers.patroni = {
+    objects = {
+      user = lib.attrNames config.patroni.users;
+      database = lib.attrNames config.patroni.databases;
+    };
+  };
+}

cluster/services/patroni/options.nix

@@ -0,0 +1,37 @@
+{ lib, ... }:
+
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) attrsOf enum listOf submodule str;
+in
+
+{
+  options.patroni = {
+    databases = mkOption {
+      type = attrsOf (submodule ({ name, ... }: {
+        options = {
+          owner = mkOption {
+            type = str;
+            default = name;
+          };
+        };
+      }));
+    };
+    users = mkOption {
+      type = attrsOf (submodule ({ ... }: {
+        options = {
+          locksmith = {
+            nodes = mkOption {
+              type = listOf str;
+              default = [];
+            };
+            format = mkOption {
+              type = enum [ "pgpass" "envFile" ];
+              default = "pgpass";
+            };
+          };
+        };
+      }));
+    };
+  };
+}

cluster/services/storage/default.nix

@@ -7,6 +7,8 @@ in
 {
   imports = [
     ./options.nix
+    ./incandescence.nix
+    ./simulacrum/test-data.nix
   ];
 
   services.storage = {
@@ -52,7 +54,7 @@ in
     };
     simulacrum = {
       enable = true;
-      deps = [ "wireguard" "consul" "locksmith" ];
+      deps = [ "consul" "locksmith" "incandescence" "patroni" "ways" ];
       settings = ./test.nix;
     };
   };
@@ -93,7 +95,10 @@ in
   };
 
   garage = {
-    keys.storage-prophet = {};
+    keys.storage-prophet.locksmith = {
+      nodes = [ "prophet" ];
+      format = "s3ql";
+    };
     buckets.storage-prophet = {
       allow.storage-prophet = [ "read" "write" ];
     };

cluster/services/storage/external.nix

@@ -8,7 +8,7 @@ in
   services.external-storage = {
     fileSystems.external = {
       mountpoint = "/srv/storage";
-      authFile = ./secrets/external-storage-auth-${hostName}.age;
+      locksmithSecret = "garage-storage-${hostName}";
       backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
       backendOptions = [ "disable-expect100" ];
     };

cluster/services/storage/garage-options.nix

@@ -26,62 +26,6 @@ let
         sleep 1
       done
     }
-    # FIXME: returns bogus empty string when one of the lists is empty
-    diffAdded() {
-      comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
-    }
-    diffRemoved() {
-      comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
-    }
-    # FIXME: this does not handle list items with spaces
-    listKeys() {
-      garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
-    }
-    ensureKeys() {
-      old="$(listKeys)"
-      if [[ -z "$1" ]]; then
-        for key in $old; do
-          garage key delete --yes "$key"
-        done
-      elif [[ -z "$old" ]]; then
-        for key in $1; do
-          # don't print secret key
-          garage key new --name "$key" >/dev/null
-          echo Key "$key" was created.
-        done
-      else
-        diffAdded "$old" "$1" | while read key; do
-          # don't print secret key
-          garage key new --name "$key" >/dev/null
-          echo Key "$key" was created.
-        done
-        diffRemoved "$old" "$1" | while read key; do
-          garage key delete --yes "$key"
-        done
-      fi
-    }
-    listBuckets() {
-      garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
-    }
-    ensureBuckets() {
-      old="$(listBuckets)"
-      if [[ -z "$1" ]]; then
-        for bucket in $old; do
-          garage bucket delete --yes "$bucket"
-        done
-      elif [[ -z "$old" ]]; then
-        for bucket in $1; do
-          garage bucket create "$bucket"
-        done
-      else
-        diffAdded "$old" "$1" | while read bucket; do
-          garage bucket create "$bucket"
-        done
-        diffRemoved "$old" "$1" | while read bucket; do
-          garage bucket delete --yes "$bucket"
-        done
-      fi
-    }
   '';
 in
 
@@ -203,9 +147,7 @@ in
           garage layout apply --version 1
         '';
       };
-      garage-apply = {
-        distributed.enable = true;
-        wantedBy = [ "garage.service" "multi-user.target" ];
+      garage-ready = {
         wants = [ "garage.service" ];
         after = [ "garage.service" "garage-layout-init.service" ];
         path = [ config.services.garage.package ];
@@ -219,54 +161,70 @@ in
         script = ''
           source ${garageShellLibrary}
           waitForGarageOperational
-
-          ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
-          ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
-
-          # key permissions
-          ${lib.pipe cfg.keys [
-            (lib.mapAttrsToList (key: kCfg: ''
-              garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
-            ''))
-            (lib.concatStringsSep "\n")
-          ]}
-
-          # bucket permissions
-          ${lib.pipe cfg.buckets [
-            (lib.mapAttrsToList (bucket: bCfg:
-              lib.mapAttrsToList (key: perms: ''
-                garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
-                garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
-              '') bCfg.allow
-            ))
-            lib.flatten
-            (lib.concatStringsSep "\n")
-          ]}
-
-          # bucket quotas
-          ${lib.pipe cfg.buckets [
-            (lib.mapAttrsToList (bucket: bCfg: ''
-              garage bucket set-quotas '${bucket}' \
-                --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
-                --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
-            ''))
-            (lib.concatStringsSep "\n")
-          ]}
-
-          # bucket website access
-          ${lib.pipe cfg.buckets [
-            (lib.mapAttrsToList (bucket: bCfg: ''
-              garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
-            ''))
-            (lib.concatStringsSep "\n")
-          ]}
         '';
       };
     };
 
+    services.incandescence.providers.garage = {
+      locksmith = true;
+      wantedBy = [ "garage.service" "multi-user.target" ];
+      partOf = [ "garage.service" ];
+      wants = [ "garage-ready.service" ];
+      after = [ "garage-ready.service" ];
+
+      packages = [
+        config.services.garage.package
+      ];
+      formulae = {
+        key = {
+          destroyAfterDays = 0;
+          create = key: ''
+            # don't print secret key
+            garage key new --name ${lib.escapeShellArg key} >/dev/null
+            echo Key ${lib.escapeShellArg key} was created.
+          '';
+          destroy = ''
+            garage key delete --yes "$OBJECT"
+          '';
+          change = key: let
+            kCfg = cfg.keys.${key};
+          in ''
+            garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
+          '';
+        };
+        bucket = {
+          deps = [ "key" ];
+          destroyAfterDays = 30;
+          create = bucket: ''
+            garage bucket create ${lib.escapeShellArg bucket}
+          '';
+          destroy = ''
+            garage bucket delete --yes "$OBJECT"
+          '';
+          change = bucket: let
+            bCfg = cfg.buckets.${bucket};
+          in ''
+            # permissions
+            ${lib.concatStringsSep "\n" (lib.flatten (
+              lib.mapAttrsToList (key: perms: ''
+                garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
+                garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
+              '') bCfg.allow
+            ))}
+
+            # quotas
+            garage bucket set-quotas ${lib.escapeShellArg bucket} \
+              --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
+              --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
+
+            # website access
+            garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
+          '';
+        };
+      };
+    };
+
     services.locksmith.providers.garage = {
-      wantedBy = [ "garage-apply.service" ];
-      after = [ "garage-apply.service" ];
       secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
         common = {
           inherit (kCfg.locksmith) mode owner group nodes;

cluster/services/storage/incandescence.nix

@@ -0,0 +1,10 @@
+{ config, lib, ... }:
+
+{
+  incandescence.providers.garage = {
+    objects = {
+      key = lib.attrNames config.garage.keys;
+      bucket = lib.attrNames config.garage.buckets;
+    };
+  };
+}

cluster/services/storage/secrets/external-storage-auth-prophet.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
-2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
--> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
-0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
--> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
-LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
---- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
-™ªg¬NÑ8´¨\K!p
«ï…7ù¶›käõ¯#ŒÏu›µ*{}Tþ0·|@ÉÿàE>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†‡4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
-µÿÂ9gÏi"Q
–ÿ
-™›Ù®à

cluster/services/storage/simulacrum/test-data.nix

@@ -0,0 +1,8 @@
+{ config, lib, ... }:
+
+{
+  garage = lib.mkIf config.simulacrum {
+    keys.testkey = {};
+    buckets.testbucket.allow.testKey = [ "read" "write" ];
+  };
+}

cluster/services/storage/test.nix

@@ -18,19 +18,6 @@ in
       configuration = {
         services.garage = {
           layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
-          keys.testKey.allow.createBucket = true;
-          buckets = {
-            bucket1 = {
-              allow.testKey = [ "read" "write" ];
-              quotas = {
-                maxObjects = 300;
-                maxSize = 400 * 1024 * 1024;
-              };
-            };
-            bucket2 = {
-              allow.testKey = [ "read" ];
-            };
-          };
         };
         system.ascensions.garage-layout.incantations = lib.mkForce (i: [
           (i.runGarage ''
@@ -68,10 +55,15 @@ in
         assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
         assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
 
+    consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
+    addr = consulConfig["addresses"]["http"]
+    port = consulConfig["ports"]["http"]
+    setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
     with subtest("should apply new layout from scratch"):
       for node in nodes:
         node.systemctl("stop garage.service")
         node.succeed("rm -rf /var/lib/garage-metadata")
+      garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
 
       for node in nodes:
         node.systemctl("start garage.service")
@@ -85,21 +77,28 @@ in
       for node in nodes:
         node.wait_until_succeeds("garage layout show | grep -w 2000")
         assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
-        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+        assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
 
     with subtest("should create specified buckets and keys"):
       for node in nodes:
-        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
-      garage1.succeed("garage key list | grep testKey")
-      garage1.succeed("garage bucket list | grep bucket1")
-      garage1.succeed("garage bucket list | grep bucket2")
+        node.wait_for_unit("incandescence-garage.target")
+      garage1.succeed("garage key list | grep testkey")
+      garage1.succeed("garage bucket list | grep testbucket")
 
-    with subtest("should delete unspecified buckets and keys"):
+    with subtest("should delete unspecified keys"):
       garage1.succeed("garage bucket create unwantedbucket")
       garage1.succeed("garage key new --name unwantedkey")
-      garage1.succeed("systemctl restart garage-apply.service")
-
+      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
+      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
+      garage1.succeed("systemctl restart garage.service")
+      garage1.wait_for_unit("incandescence-garage.target")
       garage1.fail("garage key list | grep unwantedkey")
+      garage1.succeed("garage bucket list | grep unwantedbucket")
+
+    with subtest("should delete unspecified buckets after grace period"):
+      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
+      garage1.succeed("systemctl restart garage.service")
+      garage1.wait_for_unit("incandescence-garage.target")
       garage1.fail("garage bucket list | grep unwantedbucket")
   '';
 }

cluster/services/ways/default.nix

@@ -8,6 +8,7 @@
   services.ways = {
     nodes.host = config.services.websites.nodes.host;
     nixos.host = ./host.nix;
+    simulacrum.deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
   };
 
   dns.records = lib.mapAttrs'

cluster/services/ways/host.nix

@@ -69,11 +69,16 @@ in
         {
           source = let
             upstreams = lib.mapAttrsToList (_: cfg: ''
+              {{ if ne (len (service "${cfg.consulService}~_agent")) 0 }}
+              # ${cfg.consulService}
               upstream ${cfg.nginxUpstreamName} {
                 {{ range $i, $e := service "${cfg.consulService}~_agent" -}}
                 server {{ .Address }}:{{ .Port }}{{ if ne $i 0 }} backup{{ end }};
-                {{end}}
+                {{ end }}
               }
+              {{ else }}
+              # upstream ${cfg.nginxUpstreamName} (${cfg.consulService}): no servers available
+              {{ end }}
             '') consulServiceWays;
           in pkgs.writeText "ways-upstreams.ctmpl" (lib.concatStringsSep "\n" (lib.unique upstreams));
           destination = "/run/consul-template/nginx-ways-upstreams.conf";

cluster/simulacrum/default.nix

@@ -4,9 +4,15 @@
 
 let
   serviceConfig = config.cluster.config.services.${service};
-  serviceList = [ service ] ++ serviceConfig.simulacrum.deps;
+  serviceList = getDepsRecursive [] service;
   allAugments = map (svc: config.cluster.config.services.${svc}.simulacrum.augments) serviceList;
 
+  getDepsRecursive = acc: service: let
+    deps = lib.subtractLists acc config.cluster.config.services.${service}.simulacrum.deps;
+    acc' = acc ++ [ service ];
+    recurse = getDepsRecursive acc';
+  in lib.unique (lib.flatten ([ service ] ++ map recurse deps));
+
   lift = config;
 
   snakeoil = {
@@ -17,7 +23,8 @@ let
   };
 
   nodes = lib.attrNames config.gods.fromLight;
-  digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes (lib.range 1 255));
+  nodes' = lib.attrNames (config.gods.fromLight // { nowhere = null; });
+  digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes' (lib.range 1 255));
   depot' = extendModules {
     modules = [
       ({ config, ... }: {
@@ -51,6 +58,12 @@ testers.runNixOSTest {
 
   imports = [
     serviceConfig.simulacrum.settings
+    ./nowhere
+    {
+      nodes.nowhere.imports = [
+        config.flake.nixosModules.port-magic
+      ];
+    }
   ] ++ allAugments;
 
   _module.args = {
@@ -73,12 +86,15 @@ testers.runNixOSTest {
         ${hour.interfaces.primary.link} = {
           useDHCP = lib.mkForce false;
           virtual = true;
-          ipv4.addresses = lib.mkForce [
+          ipv4.addresses = lib.mkForce ([
             {
               address = hour.interfaces.primary.addr;
               prefixLength = 32;
             }
-          ];
+          ] ++ lib.optional hour.interfaces.primary.isNat {
+            address = hour.interfaces.primary.addrPublic;
+            prefixLength = 32;
+          });
         };
         eth1.ipv4.routes = lib.pipe nodes [
           (lib.filter (n: n != node))
@@ -95,6 +111,7 @@ testers.runNixOSTest {
       firewall.extraCommands = lib.mkAfter (lib.optionalString (hour.interfaces.primary.isNat) ''
         # self-nat
         iptables -t nat -A PREROUTING -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
+        iptables -t nat -A OUTPUT -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
         iptables -t nat -A POSTROUTING -s ${hour.interfaces.primary.addr} -j SNAT --to-source ${hour.interfaces.primary.addrPublic}
       '');
     };

cluster/simulacrum/nowhere/default.nix

@@ -0,0 +1,101 @@
+{ cluster, config, lib, pkgs, ... }:
+
+let
+  lift = config;
+
+  cfsslConfigIntermediateCA = pkgs.writeText "simulacrum-cfssl-config.json" (builtins.toJSON {
+    signing = {
+      default.expiry = "8760h";
+      profiles.intermediate = {
+        expiry = "8760h";
+        usages = [
+          "cert sign"
+          "crl sign"
+        ];
+        ca_constraint = {
+          is_ca = true;
+          max_path_len = 1;
+        };
+      };
+    };
+  });
+
+  caCsr = pkgs.writeText "simulacrum-ca-csr.json" (builtins.toJSON {
+    CN = "Simulacrum Root CA";
+  });
+
+  ca = pkgs.runCommand "simulacrum-snakeoil-ca" {
+    nativeBuildInputs = [
+      pkgs.cfssl
+    ];
+  } ''
+    mkdir $out
+    cfssl gencert --initca ${caCsr} | cfssljson --bare $out/ca
+  '';
+
+  genCert = extraFlags: csrData: let
+    csr = pkgs.writeText "simulacrum-csr.json" (builtins.toJSON csrData);
+  in pkgs.runCommand "simulacrum-snakeoil-cert" {
+    nativeBuildInputs = [
+      pkgs.cfssl
+    ];
+  } ''
+    mkdir $out
+    cfssl gencert ${lib.escapeShellArgs ([
+      "--ca=file:${ca}/ca.pem"
+      "--ca-key=file:${ca}/ca-key.pem"
+    ] ++ extraFlags ++ [
+      csr
+    ])} | cfssljson --bare $out/cert
+  '';
+
+  genHostCert = hostname: genCert [ "--hostname=${hostname}" ] { CN = hostname; };
+
+  getNodeAddr = node: (builtins.head config.nodes.${node}.networking.interfaces.eth1.ipv4.addresses).address;
+in
+
+{
+  imports = [
+    ./options.nix
+  ];
+  defaults = {
+    networking.hosts."${getNodeAddr "nowhere"}" = lib.attrNames config.nowhere.names;
+    security.pki.certificateFiles = [
+      "${ca}/ca.pem"
+    ];
+  };
+
+  nowhere.certs = {
+    inherit ca;
+    intermediate = genCert [ "--config=${cfsslConfigIntermediateCA}" "--profile=intermediate" ] {
+      CN = "Simulacrum Intermediate CA";
+    };
+  };
+
+  nodes.nowhere = { config, depot, ... }: {
+    networking = {
+      firewall.allowedTCPPorts = [ 443 ];
+      interfaces.eth1.ipv4.routes = lib.mapAttrsToList (name: hour: {
+        address = hour.interfaces.primary.addrPublic;
+        prefixLength = 32;
+        via = getNodeAddr name;
+      }) depot.gods.fromLight;
+      nameservers = map (name: depot.hours.${name}.interfaces.primary.addrPublic) cluster.config.services.dns.nodes.authoritative;
+    };
+    services.nginx = {
+      enable = true;
+      recommendedProxySettings = true;
+      virtualHosts = lib.mapAttrs (name: link: let
+        cert = genHostCert name;
+      in {
+        forceSSL = true;
+        sslCertificate = "${cert}/cert.pem";
+        sslCertificateKey = "${cert}/cert-key.pem";
+        locations."/" = {
+          proxyPass = config.links.${link}.url;
+          extraConfig = "proxy_ssl_verify off;";
+        };
+      }) lift.nowhere.names;
+    };
+  };
+}

cluster/simulacrum/nowhere/options.nix

@@ -0,0 +1,16 @@
+{ lib, ... }:
+
+{
+  options.nowhere = {
+    names = lib.mkOption {
+      description = "Hostnames that point Nowhere.";
+      type = with lib.types; attrsOf str;
+      default = {};
+    };
+    certs = lib.mkOption {
+      description = "Snakeoil certificate packages.";
+      type = with lib.types; attrsOf package;
+      default = {};
+    };
+  };
+}

lib/nginx.nix

@@ -36,7 +36,8 @@
 
         proxyGhost = scheme: target: basic // {
           locations."/".extraConfig = ''
-            proxy_pass ${scheme}://${target};
+            set $nix_proxy_ghost_target "${scheme}://${target}";
+            proxy_pass $nix_proxy_ghost_target;
             proxy_set_header Host ${target};
             proxy_set_header Referer ${scheme}://${target};
             proxy_cookie_domain ${target} domain.invalid;

modules/external-storage/default.nix

@@ -109,7 +109,13 @@ in
         value = let
           isUnderlay = fs.underlay != null;
 
-          fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);
+          backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
+
+          fsType = if isUnderlay then "local" else lib.head backendParts;
+
+          s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
+
+          s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
 
           localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
         in {
@@ -132,8 +138,12 @@ in
             ExecStartPre = map lib.escapeShellArgs [
               [
                 (let
+                  authFile = if fs.locksmithSecret != null then
+                    "/run/locksmith/${fs.locksmithSecret}"
+                  else
+                    cfgAge.secrets."storageAuth-${name}".path;
                   mkfsEncrypted = ''
-                    ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
+                    ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
                       | cut -d' ' -f2- \
                       | ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
                   '';
@@ -144,6 +154,11 @@ in
 
                   detectFs = {
                     local = "test -e ${localBackendPath}/s3ql_metadata";
+                    s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
+                      export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
+                      export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
+                      ${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
+                    '';
                   }.${fsType} or null;
                 in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
                   if ! ${detectFs}; then