cluster/services/forge: switch to locksmith secrets

cluster/secrets/forge-s3AccessKeyID.age

@@ -1,9 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A 5NtIVE60zj6mR2+/2N0eS6lWTkddt3rsDWHZpNefLAo
-5b8sLEf76HReLUuBcTVjTOnzjrVdwcxnG0TraO+eHww
--> ssh-ed25519 5/zT0w RbikYmV32iG1QgMDiObNPV+GZOW35K6hbx2n2eLCvno
-bXVeCmC2UpnTx8Udpx657mMGqRvYO7Gn53YwtW6NJEk
--> ssh-ed25519 d3WGuA 4+sPg6CCmOxlJUls3qZpWvN+f2V4SHRXhrBxKQPQyho
-z2TCvvpOZ8Nh4IQ0oPKD1yj0dP3rnLMzuvRpZxE2SSU
---- aj9laXQ3ccpGvhDpYIrpPzxfC4G6A5LdCkaWFSgUXUY
-0žÜ¾KÿWðúÉ=þ,nÃÑðŽ—½O{9Z±HÇN\—ûwšᇎ#›•Ù´gYÊD¬PåJÿÀ

cluster/services/forge/default.nix

@@ -14,8 +14,6 @@
         owner = "forgejo";
       };
       dbCredentials.nodes = server;
-      s3AccessKeyID.nodes = server;
-      s3SecretAccessKey.nodes = server;
     };
   };
 
@@ -24,7 +22,7 @@
   in config.hostLinks.${host}.forge.url;
 
   garage = {
-    keys.forgejo = { };
+    keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
     buckets.forgejo.allow.forgejo = [ "read" "write" ];
   };
 }

cluster/services/forge/server.nix

@@ -23,6 +23,11 @@ in
     ];
   };
 
+  services.locksmith.waitForSecrets.forgejo = [
+    "garage-forgejo-id"
+    "garage-forgejo-secret"
+  ];
+
   services.forgejo = {
     enable = true;
     package = depot.packages.forgejo;
@@ -73,8 +78,8 @@ in
     };
     secrets = {
       storage = {
-        MINIO_ACCESS_KEY_ID = secrets.s3AccessKeyID.path;
-        MINIO_SECRET_ACCESS_KEY = secrets.s3SecretAccessKey.path;
+        MINIO_ACCESS_KEY_ID = "/run/locksmith/garage-forgejo-id";
+        MINIO_SECRET_ACCESS_KEY = "/run/locksmith/garage-forgejo-secret";
       };
     };
   };

cluster/services/locksmith/default.nix

@@ -2,9 +2,17 @@
 
 {
   services.locksmith = {
-    nodes.receiver = config.services.consul.nodes.agent;
-    nixos.receiver = [
-      ./receiver.nix
-    ];
+    nodes = {
+      receiver = config.services.consul.nodes.agent;
+      provider = config.services.consul.nodes.agent;
+    };
+    nixos = {
+      receiver = [
+        ./receiver.nix
+      ];
+      provider = [
+        ./provider.nix
+      ];
+    };
   };
 }

cluster/services/locksmith/provider.nix

@@ -0,0 +1,102 @@
+{ config, depot, lib, pkgs, ... }:
+
+let
+  kvRoot = "secrets/locksmith";
+  activeProvders = lib.filterAttrs (_: cfg: lib.any (secret: secret.nodes != []) (lib.attrValues cfg.secrets)) config.services.locksmith.providers;
+in
+
+{
+  options.services.locksmith = with lib; {
+    providers = mkOption {
+      type = types.attrsOf (types.submodule ({ ... }: {
+        options = {
+          wantedBy = mkOption {
+            type = types.listOf types.str;
+            default = [];
+          };
+          after = mkOption {
+            type = types.listOf types.str;
+            default = [];
+          };
+          secrets = mkOption {
+            type = types.attrsOf (types.submodule ({ ... }: {
+              options = {
+                nodes = mkOption {
+                  type = types.listOf types.str;
+                  default = [];
+                };
+                command = mkOption {
+                  type = types.coercedTo types.package (package: "${package}") types.str;
+                };
+                owner = mkOption {
+                  type = types.str;
+                  default = "root";
+                };
+                group = mkOption {
+                  type = types.str;
+                  default = "root";
+                };
+                mode = mkOption {
+                  type = types.str;
+                  default = "0400";
+                };
+              };
+            }));
+          };
+        };
+      }));
+    };
+  };
+
+  config.systemd.services = lib.mapAttrs' (providerName: providerConfig: {
+    name = "locksmith-provider-${providerName}";
+    value = let
+      providerRoot = "${kvRoot}/${providerName}";
+    in {
+      description = "Locksmith Provider | ${providerName}";
+      distributed.enable = true;
+      inherit (providerConfig) wantedBy after;
+      serviceConfig = {
+        Type = "oneshot";
+        PrivateTmp = true;
+        LoadCredential = lib.mkForce [];
+      };
+      path = [
+        config.services.consul.package
+        pkgs.age
+      ];
+      script = let
+        activeSecrets = lib.filterAttrs (_: secret: secret.nodes != []) providerConfig.secrets;
+        activeNodes = lib.unique (lib.flatten (lib.mapAttrsToList (_: secret: secret.nodes) activeSecrets));
+        secretNames = map (name: "${providerRoot}-${name}/") (lib.attrNames activeSecrets);
+
+        createSecret = { path, nodes, owner, mode, group, command }: ''
+          consul kv put ${lib.escapeShellArg path}/mode ${lib.escapeShellArg mode}
+          consul kv put ${lib.escapeShellArg path}/owner ${lib.escapeShellArg owner}
+          consul kv put ${lib.escapeShellArg path}/group ${lib.escapeShellArg group}
+          ${lib.concatStringsSep "\n" (map (node: ''
+            consul kv put ${lib.escapeShellArg path}/recipient/${node} "$( (${command}) | age --encrypt --armor -r ${lib.escapeShellArg depot.hours.${node}.ssh.id.publicKey})"
+          '') nodes)}
+        '';
+      in ''
+        # create/update secrets
+        ${lib.pipe activeSecrets [
+          (lib.mapAttrsToList (secretName: secretConfig: createSecret {
+            path = "${providerRoot}-${secretName}";
+            inherit (secretConfig) nodes mode owner group command;
+          }))
+          (lib.concatStringsSep "\n")
+        ]}
+
+        # delete leftover secrets of this provider
+        consul kv get --keys '${providerRoot}-' | grep -v ${lib.concatStringsSep " \\\n  " (map (secret: "-e ${lib.escapeShellArg secret}") secretNames)} | xargs --no-run-if-empty -n1 consul kv delete --recurse
+
+        # notify
+        ${lib.pipe activeNodes [
+          (map (node: "consul event --name=chant:locksmith --node=${node}"))
+          (lib.concatStringsSep "\n")
+        ]}
+      '';
+    };
+  }) activeProvders;
+}

cluster/services/locksmith/receiver.nix

@@ -10,44 +10,80 @@ let
 in
 
 {
-  systemd.tmpfiles.settings.locksmith = {
-    "/run/locksmith".d = {
-      mode = "0711";
-    };
+  options.services.locksmith.waitForSecrets = lib.mkOption {
+    type = with lib.types; attrsOf (listOf str);
   };
 
-  systemd.services.locksmith = {
-    description = "The Locksmith's Chant";
-    wantedBy = [ "multi-user.target" ];
-    wants = [ "consul.service" ];
-    after = [ "consul.service" ];
-    chant.enable = true;
-    path = [
-      config.services.consul.package
-    ];
-    environment = {
-      CONSUL_HTTP_ADDR = consulHttpAddr;
-    };
-    serviceConfig = {
-      PrivateTmp = true;
-      WorkingDirectory = "/tmp";
-      IPAddressDeny = [ "any" ];
-      IPAddressAllow = [ consulIpAddr ];
-      LoadCredential = lib.mkForce [];
-    };
-    script = ''
-      consul kv get --keys ${kvRoot}/ | ${pkgs.gnused}/bin/sed 's,/$,,g' | while read secret; do
-        out="$(mktemp -u /run/locksmith/.locksmith-secret.XXXXXXXXXXXXXXXX)"
-        if [[ "$(consul kv get --keys "$secret/${kvValue}")" == "$secret/${kvValue}" ]]; then
-          owner="$(consul kv get "$secret/owner")"
-          group="$(consul kv get "$secret/group")"
-          mode="$(consul kv get "$secret/mode")"
-          consul kv get "$secret/${kvValue}" | ${pkgs.age}/bin/age --decrypt -i /etc/ssh/ssh_host_ed25519_key -o $out
-          chown -v "$owner:$group" $out
-          chmod -v "$mode" $out
-          mv -v $out "/run/locksmith/$(basename "$secret")"
-        fi
-      done
-    '';
-  };
+  config = lib.mkMerge [
+    {
+      systemd.services = lib.mapAttrs' (name: secrets: {
+        name = "locksmith-wait-secrets-${name}";
+        value = {
+          description = "Wait for secrets: ${name}";
+          after = [ "locksmith.service" ];
+          before = [ "${name}.service" ];
+          requiredBy = [ "${name}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            IPAddressDeny = [ "any" ];
+          };
+          path = [
+            pkgs.inotify-tools
+          ];
+          script = ''
+            for key in ${lib.escapeShellArgs secrets}; do
+              if ! test -e "$key"; then
+                echo "Waiting for secret: $key"
+                inotifywait -qq -e create,moved_to --include "$key" /run/locksmith
+              fi
+              echo "Heard secret: $key"
+            done
+            echo "All secrets known."
+          '';
+        };
+      }) config.services.locksmith.waitForSecrets;
+    }
+    {
+      systemd.tmpfiles.settings.locksmith = {
+        "/run/locksmith".d = {
+          mode = "0711";
+        };
+      };
+
+      systemd.services.locksmith = {
+        description = "The Locksmith's Chant";
+        wantedBy = [ "multi-user.target" ];
+        wants = [ "consul.service" ];
+        after = [ "consul.service" ];
+        chant.enable = true;
+        path = [
+          config.services.consul.package
+        ];
+        environment = {
+          CONSUL_HTTP_ADDR = consulHttpAddr;
+        };
+        serviceConfig = {
+          PrivateTmp = true;
+          WorkingDirectory = "/tmp";
+          IPAddressDeny = [ "any" ];
+          IPAddressAllow = [ consulIpAddr ];
+          LoadCredential = lib.mkForce [];
+        };
+        script = ''
+          consul kv get --keys ${kvRoot}/ | ${pkgs.gnused}/bin/sed 's,/$,,g' | while read secret; do
+            out="$(mktemp -u /run/locksmith/.locksmith-secret.XXXXXXXXXXXXXXXX)"
+            if [[ "$(consul kv get --keys "$secret/${kvValue}")" == "$secret/${kvValue}" ]]; then
+              owner="$(consul kv get "$secret/owner")"
+              group="$(consul kv get "$secret/group")"
+              mode="$(consul kv get "$secret/mode")"
+              consul kv get "$secret/${kvValue}" | ${pkgs.age}/bin/age --decrypt -i /etc/ssh/ssh_host_ed25519_key -o $out
+              chown -v "$owner:$group" $out
+              chmod -v "$mode" $out
+              mv -v $out "/run/locksmith/$(basename "$secret")"
+            fi
+          done
+        '';
+      };
+    }
+  ];
 }

cluster/services/storage/garage-options.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, depot, lib, pkgs, ... }:
 
 let
   cfg = config.services.garage;
@@ -102,11 +102,37 @@ in
     };
     keys = mkOption {
       type = with types; attrsOf (submodule {
-        options.allow = {
-          createBucket = mkOption {
-            description = "Allow the key to create new buckets.";
-            type = bool;
-            default = false;
+        options = {
+          allow = {
+            createBucket = mkOption {
+              description = "Allow the key to create new buckets.";
+              type = bool;
+              default = false;
+            };
+          };
+          locksmith = {
+            nodes = mkOption {
+              description = "Nodes that this key will be made available to via Locksmith.";
+              type = listOf str;
+              default = [];
+            };
+            format = mkOption {
+              description = "Locksmith secret format.";
+              type = enum [ "files" "aws" "envFile" ];
+              default = "files";
+            };
+            owner = mkOption {
+              type = str;
+              default = "root";
+            };
+            group = mkOption {
+              type = str;
+              default = "root";
+            };
+            mode = mkOption {
+              type = str;
+              default = "0400";
+            };
           };
         };
       });
@@ -237,5 +263,48 @@ in
         '';
       };
     };
+
+    services.locksmith.providers.garage = {
+      wantedBy = [ "garage-apply.service" ];
+      after = [ "garage-apply.service" ];
+      secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
+        common = {
+          inherit (kCfg.locksmith) mode owner group nodes;
+        };
+        getKeyID = "${cfg.package}/bin/garage key info ${lib.escapeShellArg key} | grep -m1 'Key ID:' | cut -d ' ' -f3";
+        getSecretKey = "${cfg.package}/bin/garage key info ${lib.escapeShellArg key} | grep -m1 'Secret key:' | cut -d ' ' -f3";
+      in if kCfg.locksmith.format == "files" then {
+        "${key}-id" = common // {
+          command = getKeyID;
+        };
+        "${key}-secret" = common // {
+          command = getSecretKey;
+        };
+      } else let
+        template = pkgs.writeText "garage-key-template" {
+          aws = ''
+            [default]
+            aws_access_key_id=@@GARAGE_KEY_ID@@
+            aws_secret_access_key=@@GARAGE_SECRET_KEY@@
+          '';
+          envFile = ''
+            AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
+            AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
+          '';
+        }.${kCfg.locksmith.format};
+      in {
+        ${key} = common // {
+          command = pkgs.writeShellScript "garage-render-key-template" ''
+            tmpFile="$(mktemp -ut garageKeyTemplate-XXXXXXXXXXXXXXXX)"
+            cp ${template} "$tmpFile"
+            trap "rm -f $tmpFile" EXIT
+            chmod 600 "$tmpFile"
+            ${getKeyID} | ${pkgs.replace-secret}/bin/replace-secret '@@GARAGE_KEY_ID@@' /dev/stdin "$tmpFile"
+            ${getSecretKey} | ${pkgs.replace-secret}/bin/replace-secret '@@GARAGE_SECRET_KEY@@' /dev/stdin "$tmpFile"
+            cat "$tmpFile"
+          '';
+        };
+      }) cfg.keys);
+    };
   };
 }