flake.lock: Update

Flake lock file updates: • Updated input 'agenix': 'github:ryantm/agenix/3f1dae074a12feb7327b4bf43cbac0d124488bb7' (2024-07-30) → 'github:ryantm/agenix/f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41' (2024-08-10) • Updated input 'attic': 'github:zhaofengli/attic/e127acbf9a71ebc0c26bc8e28346822e0a6e16ba' (2024-08-01) → 'github:zhaofengli/attic/26b9417bde03edc6280d1f7ce709cd619cdb72d4' (2024-08-13) • Updated input 'hercules-ci-agent': 'github:hercules-ci/hercules-ci-agent/2e10fb21fc2e07edf40763b73443e5934bd40947' (2024-07-05) → 'github:hercules-ci/hercules-ci-agent/c303cc8e437c0fd26b9452472e7df5aa374e9177' (2024-08-15) • Updated input 'nar-serve': 'github:numtide/nar-serve/9d0eff868d328fe67c60c26c8ba50e0b9d8de867' (2024-07-31) → 'github:numtide/nar-serve/846cb9d55aee9af7c7a1eaf7016f2c5b9e01cc6e' (2024-08-15) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/f2d6c7123138044e0c68902268bd8f37dd7e2fa7' (2024-08-01) → 'github:NixOS/nixpkgs/c46d43c625f161d3bd22052a5f8f9df703d81be8' (2024-08-15)
2024-08-16 02:21:12 +00:00
32 changed files with 341 additions and 726 deletions
--- a/cluster/secrets/forge-dbCredentials.age
+++ b/cluster/secrets/forge-dbCredentials.age
@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
+JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
+-> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
+2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
+-> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
+bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
+-> Ovax-grease ^1$]}H G4 FpDF XKHkj{
+IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
+--- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
+ÜA³×oÈð:±‹`ÜVd±å(Kät:fk¼’}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69i›l`ÛÆJKwAè8y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
+üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ý’wÊ
+ÆÜßÌZî!^þRˆéÿv¾…ïk‹Êp»ÛPÌ)ýÌ†ÍpÓV5²FÎ„ÆÚÙÚÞhBÇ»ßb#Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè‹}€ß	ƒ¬E}~ZHûjmyq{òxŠ–Éôß<C3B4>"”éÀ´C#šójÿÐ.ò§yÔ£¸v¦
<0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që
--- a/cluster/services/attic/default.nix
+++ b/cluster/services/attic/default.nix
@ -33,7 +33,7 @@
    };
  };

-  garage = config.lib.forService "attic" {
+  garage = {
    keys.attic.locksmith = {
      nodes = config.services.attic.nodes.server;
      owner = "atticd";
@ -48,16 +48,14 @@
    serverAddrs = map
      (node: depot.hours.${node}.interfaces.primary.addrPublic)
      config.services.attic.nodes.server;
-  in config.lib.forService "attic" {
+  in {
    cache.target = serverAddrs;
  };

-  ways = config.lib.forService "attic" {
-    cache-api = {
-      consulService = "atticd";
-      extras.extraConfig = ''
-        client_max_body_size 4G;
-      '';
-    };
+  ways.cache-api = {
+    consulService = "atticd";
+    extras.extraConfig = ''
+      client_max_body_size 4G;
+    '';
  };
 }
--- a/cluster/services/consul/test.nix
+++ b/cluster/services/consul/test.nix
@ -1,8 +1,4 @@
-{ lib, ... }:
-
 {
-  defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
-
  testScript = ''
    import json

--- a/cluster/services/forge/default.nix
+++ b/cluster/services/forge/default.nix
@ -13,36 +13,25 @@
        nodes = server;
        owner = "forgejo";
      };
+      dbCredentials.nodes = server;
    };
  };

-  ways = let
+  ways.forge.target = let
    host = builtins.head config.services.forge.nodes.server;
-  in config.lib.forService "forge" {
-    forge.target = config.hostLinks.${host}.forge.url;
-  };
+  in config.hostLinks.${host}.forge.url;

-  patroni = config.lib.forService "forge" {
-    databases.forge = {};
-    users.forge.locksmith = {
-      nodes = config.services.forge.nodes.server;
-      format = "raw";
-    };
-  };
-
-  garage = config.lib.forService "forge" {
+  garage = {
    keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
    buckets.forgejo.allow.forgejo = [ "read" "write" ];
  };

-  monitoring.blackbox.targets.forge = config.lib.forService "forge" {
+  monitoring.blackbox.targets.forge = {
    address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
    module = "https2xx";
  };

-  dns.records = config.lib.forService "forge" {
-    "ssh.forge".target = map
-      (node: depot.hours.${node}.interfaces.primary.addrPublic)
-      config.services.forge.nodes.server;
-  };
+  dns.records."ssh.forge".target = map
+    (node: depot.hours.${node}.interfaces.primary.addrPublic)
+    config.services.forge.nodes.server;
 }
--- a/cluster/services/forge/server.nix
+++ b/cluster/services/forge/server.nix
@ -26,7 +26,6 @@ in
  services.locksmith.waitForSecrets.forgejo = [
    "garage-forgejo-id"
    "garage-forgejo-secret"
-    "patroni-forge"
  ];

  services.forgejo = {
@ -40,7 +39,7 @@ in
      inherit (patroni) port;
      name = "forge";
      user = "forge";
-      passwordFile = "/run/locksmith/patroni-forge";
+      passwordFile = secrets.dbCredentials.path;
    };
    settings = {
      DEFAULT = {
--- a/cluster/services/hercules-ci-multi-agent/default.nix
+++ b/cluster/services/hercules-ci-multi-agent/default.nix
@ -62,7 +62,7 @@
      lib.unique
      (map (x: "hci-agent-${x}"))
    ];
-  in config.lib.forService "hercules-ci-multi-agent" {
+  in {
    keys = lib.genAttrs hciAgentKeys (lib.const {});
    buckets.nix-store = {
      allow = lib.genAttrs hciAgentKeys (lib.const [ "read" "write" ]);
--- a/cluster/services/monitoring/default.nix
+++ b/cluster/services/monitoring/default.nix
@ -72,7 +72,7 @@ in
    };
  };

-  garage = config.lib.forService "monitoring" {
+  garage = {
    keys = {
      loki-ingest.locksmith = {
        nodes = config.services.monitoring.nodes.logging;
@ -93,7 +93,7 @@ in
    };
  };

-  ways = config.lib.forService "monitoring" {
+  ways = {
    monitoring = {
      consulService = "grafana";
      extras.locations."/".proxyWebsockets = true;
--- a/cluster/services/monitoring/provisioning/objects/dashboards/dashboard-dc7545ef-3180-4a5e-a289-1e64571ebb87.json
+++ b/cluster/services/monitoring/provisioning/objects/dashboards/dashboard-dc7545ef-3180-4a5e-a289-1e64571ebb87.json
@ -19,23 +19,10 @@
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 0,
-    "id": 1210,
+    "id": 16,
    "links": [],
    "liveNow": false,
    "panels": [
-      {
-        "collapsed": false,
-        "gridPos": {
-          "h": 1,
-          "w": 24,
-          "x": 0,
-          "y": 0
-        },
-        "id": 7,
-        "panels": [],
-        "title": "Replication",
-        "type": "row"
-      },
      {
        "datasource": {
          "type": "prometheus",
@ -71,7 +58,7 @@
          "h": 8,
          "w": 12,
          "x": 0,
-          "y": 1
+          "y": 0
        },
        "id": 2,
        "options": {
@ -79,7 +66,6 @@
          "graphMode": "area",
          "justifyMode": "auto",
          "orientation": "auto",
-          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [
              "lastNotNull"
@ -87,11 +73,9 @@
            "fields": "",
            "values": false
          },
-          "showPercentChange": false,
-          "textMode": "auto",
-          "wideLayout": true
+          "textMode": "auto"
        },
-        "pluginVersion": "11.1.3",
+        "pluginVersion": "9.5.1",
        "targets": [
          {
            "datasource": {
@ -154,7 +138,7 @@
          "h": 8,
          "w": 4,
          "x": 12,
-          "y": 1
+          "y": 0
        },
        "id": 3,
        "options": {
@ -162,17 +146,14 @@
          "graphMode": "area",
          "justifyMode": "auto",
          "orientation": "auto",
-          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [],
            "fields": "/^instance$/",
            "values": true
          },
-          "showPercentChange": false,
-          "textMode": "auto",
-          "wideLayout": true
+          "textMode": "auto"
        },
-        "pluginVersion": "11.1.3",
+        "pluginVersion": "9.5.1",
        "targets": [
          {
            "datasource": {
@ -217,7 +198,7 @@
          "h": 8,
          "w": 8,
          "x": 16,
-          "y": 1
+          "y": 0
        },
        "id": 5,
        "options": {
@ -273,7 +254,6 @@
              "mode": "palette-classic"
            },
            "custom": {
-              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
@ -287,7 +267,6 @@
                "tooltip": false,
                "viz": false
              },
-              "insertNulls": false,
              "lineInterpolation": "smooth",
              "lineStyle": {
                "fill": "solid"
@ -325,7 +304,7 @@
          "h": 13,
          "w": 12,
          "x": 0,
-          "y": 9
+          "y": 8
        },
        "id": 1,
        "options": {
@ -369,7 +348,6 @@
              "mode": "palette-classic"
            },
            "custom": {
-              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
@ -383,7 +361,6 @@
                "tooltip": false,
                "viz": false
              },
-              "insertNulls": false,
              "lineInterpolation": "smooth",
              "lineWidth": 1,
              "pointSize": 5,
@ -421,7 +398,7 @@
          "h": 13,
          "w": 12,
          "x": 12,
-          "y": 9
+          "y": 8
        },
        "id": 4,
        "options": {
@ -451,326 +428,24 @@
        ],
        "title": "Activity",
        "type": "timeseries"
-      },
-      {
-        "collapsed": false,
-        "gridPos": {
-          "h": 1,
-          "w": 24,
-          "x": 0,
-          "y": 22
-        },
-        "id": 8,
-        "panels": [],
-        "title": "Storage",
-        "type": "row"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              }
-            },
-            "mappings": [],
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 8,
-          "x": 0,
-          "y": 23
-        },
-        "id": 6,
-        "options": {
-          "displayLabels": [
-            "percent"
-          ],
-          "legend": {
-            "calcs": [],
-            "displayMode": "table",
-            "placement": "right",
-            "showLegend": true,
-            "sortBy": "Value",
-            "sortDesc": true,
-            "values": [
-              "percent",
-              "value"
-            ]
-          },
-          "pieType": "donut",
-          "reduceOptions": {
-            "calcs": [
-              "lastNotNull"
-            ],
-            "fields": "",
-            "values": false
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "pluginVersion": "11.1.3",
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "max by(datname) (sum by(datname, instance) (pg_database_size_bytes))",
-            "fullMetaSearch": false,
-            "includeNullMetadata": false,
-            "instant": true,
-            "legendFormat": "__auto",
-            "range": false,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "Database Size",
-        "type": "piechart"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisBorderShow": false,
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 25,
-              "gradientMode": "opacity",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "insertNulls": false,
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            },
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 5,
-          "x": 8,
-          "y": 23
-        },
-        "id": 9,
-        "options": {
-          "legend": {
-            "calcs": [
-              "lastNotNull",
-              "max"
-            ],
-            "displayMode": "table",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "pluginVersion": "11.1.3",
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "pg_wal_size_bytes",
-            "format": "time_series",
-            "fullMetaSearch": false,
-            "includeNullMetadata": true,
-            "instant": false,
-            "legendFormat": "{{instance}}",
-            "range": true,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "WAL Size",
-        "type": "timeseries"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "thresholds",
-              "seriesBy": "last"
-            },
-            "custom": {
-              "axisBorderShow": false,
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 25,
-              "gradientMode": "opacity",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "insertNulls": false,
-              "lineInterpolation": "linear",
-              "lineStyle": {
-                "fill": "solid"
-              },
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "decimals": 3,
-            "fieldMinMax": false,
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                }
-              ]
-            },
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 11,
-          "x": 13,
-          "y": 23
-        },
-        "id": 10,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": false
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "max(sum by(instance) (pg_database_size_bytes))",
-            "fullMetaSearch": false,
-            "includeNullMetadata": false,
-            "instant": false,
-            "legendFormat": "__auto",
-            "range": true,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "Total DB Size",
-        "type": "timeseries"
      }
    ],
    "refresh": "5m",
-    "schemaVersion": 39,
+    "schemaVersion": 38,
+    "style": "dark",
    "tags": [],
    "templating": {
      "list": []
    },
    "time": {
-      "from": "now-24h",
+      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {},
    "timezone": "",
    "title": "PostgreSQL HA",
    "uid": "dc7545ef-3180-4a5e-a289-1e64571ebb87",
-    "version": 2,
+    "version": 7,
    "weekStart": ""
  },
  "folderId": 0,
--- a/cluster/services/storage/default.nix
+++ b/cluster/services/storage/default.nix
@ -7,8 +7,6 @@ in
 {
  imports = [
    ./options.nix
-    ./incandescence.nix
-    ./simulacrum/test-data.nix
  ];

  services.storage = {
@ -32,15 +30,11 @@ in
      heresy = [
        ./heresy.nix
        ./s3ql-upgrades.nix
-      ] ++ lib.optionals config.simulacrum [
-        ./simulacrum/snakeoil-heresy-passphrase.nix
      ];
      garage = [
        ./garage.nix
        ./garage-options.nix
        ./garage-layout.nix
-      ] ++ lib.optionals config.simulacrum [
-        ./simulacrum/snakeoil-rpc-secret.nix
      ];
      garageConfig = [
        ./garage-gateway.nix
@ -54,11 +48,6 @@ in
      garageInternal = [ ./garage-internal.nix ];
      garageExternal = [ ./garage-external.nix ];
    };
-    simulacrum = {
-      enable = true;
-      deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
-      settings = ./simulacrum/test.nix;
-    };
  };

  links = {
@ -97,10 +86,7 @@ in
  };

  garage = {
-    keys.storage-prophet.locksmith = {
-      nodes = [ "prophet" ];
-      format = "s3ql";
-    };
+    keys.storage-prophet = {};
    buckets.storage-prophet = {
      allow.storage-prophet = [ "read" "write" ];
    };
--- a/cluster/services/storage/external.nix
+++ b/cluster/services/storage/external.nix
@ -8,7 +8,7 @@ in
  services.external-storage = {
    fileSystems.external = {
      mountpoint = "/srv/storage";
-      locksmithSecret = "garage-storage-${hostName}";
+      authFile = ./secrets/external-storage-auth-${hostName}.age;
      backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
      backendOptions = [ "disable-expect100" ];
    };
--- a/cluster/services/storage/garage-options.nix
+++ b/cluster/services/storage/garage-options.nix
@ -26,6 +26,62 @@ let
        sleep 1
      done
    }
+    # FIXME: returns bogus empty string when one of the lists is empty
+    diffAdded() {
+      comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
+    }
+    diffRemoved() {
+      comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
+    }
+    # FIXME: this does not handle list items with spaces
+    listKeys() {
+      garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
+    }
+    ensureKeys() {
+      old="$(listKeys)"
+      if [[ -z "$1" ]]; then
+        for key in $old; do
+          garage key delete --yes "$key"
+        done
+      elif [[ -z "$old" ]]; then
+        for key in $1; do
+          # don't print secret key
+          garage key new --name "$key" >/dev/null
+          echo Key "$key" was created.
+        done
+      else
+        diffAdded "$old" "$1" | while read key; do
+          # don't print secret key
+          garage key new --name "$key" >/dev/null
+          echo Key "$key" was created.
+        done
+        diffRemoved "$old" "$1" | while read key; do
+          garage key delete --yes "$key"
+        done
+      fi
+    }
+    listBuckets() {
+      garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
+    }
+    ensureBuckets() {
+      old="$(listBuckets)"
+      if [[ -z "$1" ]]; then
+        for bucket in $old; do
+          garage bucket delete --yes "$bucket"
+        done
+      elif [[ -z "$old" ]]; then
+        for bucket in $1; do
+          garage bucket create "$bucket"
+        done
+      else
+        diffAdded "$old" "$1" | while read bucket; do
+          garage bucket create "$bucket"
+        done
+        diffRemoved "$old" "$1" | while read bucket; do
+          garage bucket delete --yes "$bucket"
+        done
+      fi
+    }
  '';
 in

@ -62,7 +118,7 @@ in
            };
            format = mkOption {
              description = "Locksmith secret format.";
-              type = enum [ "files" "aws" "envFile" "s3ql" ];
+              type = enum [ "files" "aws" "envFile" ];
              default = "files";
            };
            owner = mkOption {
@ -147,7 +203,9 @@ in
          garage layout apply --version 1
        '';
      };
-      garage-ready = {
+      garage-apply = {
+        distributed.enable = true;
+        wantedBy = [ "garage.service" "multi-user.target" ];
        wants = [ "garage.service" ];
        after = [ "garage.service" "garage-layout-init.service" ];
        path = [ config.services.garage.package ];
@ -161,78 +219,54 @@ in
        script = ''
          source ${garageShellLibrary}
          waitForGarageOperational
+
+          ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
+          ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
+
+          # key permissions
+          ${lib.pipe cfg.keys [
+            (lib.mapAttrsToList (key: kCfg: ''
+              garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
+            ''))
+            (lib.concatStringsSep "\n")
+          ]}
+
+          # bucket permissions
+          ${lib.pipe cfg.buckets [
+            (lib.mapAttrsToList (bucket: bCfg:
+              lib.mapAttrsToList (key: perms: ''
+                garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
+                garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
+              '') bCfg.allow
+            ))
+            lib.flatten
+            (lib.concatStringsSep "\n")
+          ]}
+
+          # bucket quotas
+          ${lib.pipe cfg.buckets [
+            (lib.mapAttrsToList (bucket: bCfg: ''
+              garage bucket set-quotas '${bucket}' \
+                --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
+                --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
+            ''))
+            (lib.concatStringsSep "\n")
+          ]}
+
+          # bucket website access
+          ${lib.pipe cfg.buckets [
+            (lib.mapAttrsToList (bucket: bCfg: ''
+              garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
+            ''))
+            (lib.concatStringsSep "\n")
+          ]}
        '';
      };
    };

-    services.incandescence.providers.garage = {
-      locksmith = true;
-      wantedBy = [ "garage.service" "multi-user.target" ];
-      partOf = [ "garage.service" ];
-      wants = [ "garage-ready.service" ];
-      after = [ "garage-ready.service" ];
-
-      packages = [
-        config.services.garage.package
-      ];
-      formulae = {
-        key = {
-          destroyAfterDays = 0;
-          create = key: ''
-            if [[ "$(garage key info ${lib.escapeShellArg key} 2>&1 >/dev/null)" == "Error: 0 matching keys" ]]; then
-              # don't print secret key
-              garage key new --name ${lib.escapeShellArg key} >/dev/null
-              echo Key ${lib.escapeShellArg key} was created.
-            else
-              echo "Key already exists, assuming ownership"
-            fi
-          '';
-          destroy = ''
-            garage key delete --yes "$OBJECT"
-          '';
-          change = key: let
-            kCfg = cfg.keys.${key};
-          in ''
-            garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
-          '';
-        };
-        bucket = {
-          deps = [ "key" ];
-          destroyAfterDays = 30;
-          create = bucket: ''
-            if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == "Error: Bucket not found" ]]; then
-              garage bucket create ${lib.escapeShellArg bucket}
-            else
-              echo "Bucket already exists, assuming ownership"
-            fi
-          '';
-          destroy = ''
-            garage bucket delete --yes "$OBJECT"
-          '';
-          change = bucket: let
-            bCfg = cfg.buckets.${bucket};
-          in ''
-            # permissions
-            ${lib.concatStringsSep "\n" (lib.flatten (
-              lib.mapAttrsToList (key: perms: ''
-                garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
-                garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
-              '') bCfg.allow
-            ))}
-
-            # quotas
-            garage bucket set-quotas ${lib.escapeShellArg bucket} \
-              --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
-              --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
-
-            # website access
-            garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
-          '';
-        };
-      };
-    };
-
    services.locksmith.providers.garage = {
+      wantedBy = [ "garage-apply.service" ];
+      after = [ "garage-apply.service" ];
      secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
        common = {
          inherit (kCfg.locksmith) mode owner group nodes;
@ -257,12 +291,6 @@ in
            AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
            AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
          '';
-          s3ql = ''
-            [s3c]
-            storage-url: s3c4://
-            backend-login: @@GARAGE_KEY_ID@@
-            backend-password: @@GARAGE_SECRET_KEY@@
-          '';
        }.${kCfg.locksmith.format};
      in {
        ${key} = common // {
--- a/cluster/services/storage/garage.nix
+++ b/cluster/services/storage/garage.nix
@ -63,8 +63,6 @@ in
  };

  systemd.services.garage = {
-    requires = [ "consul-ready.service" ];
-    after = [ "consul-ready.service" ];
    unitConfig = {
      RequiresMountsFor = [ cfg.settings.data_dir ];
    };
--- a/cluster/services/storage/heresy.nix
+++ b/cluster/services/storage/heresy.nix
@ -11,7 +11,6 @@
      unitDescription = "Heresy Filesystem";
      authFile = ./secrets/heresy-encryption-key.age;
      underlay = "heresy";
-      encrypt = true;
    };
  };
 }
--- a/cluster/services/storage/incandescence.nix
+++ b/cluster/services/storage/incandescence.nix
@ -1,10 +0,0 @@
-{ config, lib, ... }:
-
-{
-  incandescence.providers.garage = {
-    objects = {
-      key = lib.attrNames config.garage.keys;
-      bucket = lib.attrNames config.garage.buckets;
-    };
-  };
-}
--- a/cluster/services/storage/secrets/external-storage-auth-prophet.age
+++ b/cluster/services/storage/secrets/external-storage-auth-prophet.age
@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
+2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
+-> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
+0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
+-> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
+LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
+--- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
+™ªg¬NÑ8´¨\K!p
«ï…7ù¶›käõ¯#ŒÏu›µ*{}Tþ0·|@ÉÿàE>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†‡4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
+µÿÂ9gÏi"Q
–ÿ
+™›Ù®à
--- a/cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix
+++ b/cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix
@ -1,8 +0,0 @@
-{
-  environment.etc."dummy-secrets/storageAuth-heresy".text = ''
-    [local]
-    storage-url: local://
-    fs-passphrase: simulacrum
-  '';
-}
-
--- a/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix
+++ b/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix
@ -1,3 +0,0 @@
-{
-  environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-}
--- a/cluster/services/storage/simulacrum/test-data.nix
+++ b/cluster/services/storage/simulacrum/test-data.nix
@ -1,8 +0,0 @@
-{ config, lib, ... }:
-
-{
-  garage = lib.mkIf config.simulacrum {
-    keys.testkey = {};
-    buckets.testbucket.allow.testKey = [ "read" "write" ];
-  };
-}
--- a/cluster/services/storage/simulacrum/test.nix
+++ b/cluster/services/storage/simulacrum/test.nix
@ -1,105 +0,0 @@
-{ cluster, lib, ... }:
-
-let
-  inherit (cluster.config.services.storage) nodes;
-
-  firstGarageNode = lib.elemAt nodes.garage 0;
-in
-
-{
-  nodes = lib.genAttrs nodes.garage (node: {
-    services.garage = {
-      layout.initial = lib.genAttrs nodes.garage (_: {
-        capacity = lib.mkOverride 51 1000;
-      });
-    };
-    specialisation.modifiedLayout = {
-      inheritParentConfig = true;
-      configuration = {
-        services.garage = {
-          layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
-        };
-        system.ascensions.garage-layout.incantations = lib.mkForce (i: [
-          (i.runGarage ''
-            garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
-            garage layout apply --version 2
-          '')
-        ]);
-      };
-    };
-  });
-
-  testScript = ''
-    import json
-    nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
-    garage1 = nodes[0]
-
-    start_all()
-
-    with subtest("should bootstrap new cluster"):
-      for node in nodes:
-        node.wait_for_unit("garage.service")
-
-      for node in nodes:
-        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
-
-    with subtest("should apply new layout with ascension"):
-      for node in nodes:
-        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
-
-      for node in nodes:
-        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
-
-      for node in nodes:
-        node.wait_until_succeeds("garage layout show | grep -w 2000")
-        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
-        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
-
-    consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
-    addr = consulConfig["addresses"]["http"]
-    port = consulConfig["ports"]["http"]
-    setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
-    with subtest("should apply new layout from scratch"):
-      for node in nodes:
-        node.systemctl("stop garage.service")
-        node.succeed("rm -rf /var/lib/garage-metadata")
-      garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
-
-      for node in nodes:
-        node.systemctl("start garage.service")
-
-      for node in nodes:
-        node.wait_for_unit("garage.service")
-
-      for node in nodes:
-        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
-
-      for node in nodes:
-        node.wait_until_succeeds("garage layout show | grep -w 2000")
-        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
-        assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
-
-    with subtest("should create specified buckets and keys"):
-      for node in nodes:
-        node.wait_for_unit("incandescence-garage.target")
-      garage1.succeed("garage key list | grep testkey")
-      garage1.succeed("garage bucket list | grep testbucket")
-
-    with subtest("should delete unspecified keys"):
-      garage1.succeed("garage bucket create unwantedbucket")
-      garage1.succeed("garage key new --name unwantedkey")
-      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
-      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
-      garage1.succeed("systemctl restart garage.service")
-      garage1.wait_for_unit("incandescence-garage.target")
-      garage1.fail("garage key list | grep unwantedkey")
-      garage1.succeed("garage bucket list | grep unwantedbucket")
-
-    with subtest("should delete unspecified buckets after grace period"):
-      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
-      garage1.succeed("systemctl restart garage.service")
-      garage1.wait_for_unit("incandescence-garage.target")
-      garage1.fail("garage bucket list | grep unwantedbucket")
-  '';
-}
-
--- a/cluster/services/ways/default.nix
+++ b/cluster/services/ways/default.nix
@ -3,17 +3,11 @@
 {
  imports = [
    ./options
-    ./simulacrum/test-data.nix
  ];

  services.ways = {
    nodes.host = config.services.websites.nodes.host;
    nixos.host = ./host.nix;
-    simulacrum = {
-      enable = true;
-      deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
-      settings = ./simulacrum/test.nix;
-    };
  };

  dns.records = lib.mapAttrs'
--- a/cluster/services/ways/simulacrum/test-data.nix
+++ b/cluster/services/ways/simulacrum/test-data.nix
@ -1,11 +0,0 @@
-{ config, lib, ... }:
-{
-  ways = lib.mkIf config.simulacrum {
-    ways-test-simple = config.lib.forService "ways" {
-      target = "http://nowhere";
-    };
-    ways-test-consul = config.lib.forService "ways" {
-      consulService = "ways-test-service";
-    };
-  };
-}
--- a/cluster/services/ways/simulacrum/test.nix
+++ b/cluster/services/ways/simulacrum/test.nix
@ -1,55 +0,0 @@
-{ cluster, config, lib, ... }:
-
-let
-  inherit (cluster._module.specialArgs.depot.lib.meta) domain;
-in
-
-{
-  nodes = lib.mkMerge [
-    {
-      nowhere = { pkgs, ... }: {
-        networking.firewall.allowedTCPPorts = [ 8080 ];
-        systemd.services.ways-simple-service = let
-          webroot = pkgs.writeTextDir "example.txt" "hello world";
-        in {
-          wantedBy = [ "multi-user.target" ];
-          serviceConfig = {
-            ExecStart = "${pkgs.darkhttpd}/bin/darkhttpd ${webroot} --port 8080";
-            DynamicUser = true;
-          };
-        };
-      };
-    }
-    (lib.genAttrs cluster.config.services.ways.nodes.host (lib.const {
-      services.nginx.upstreams.nowhere.servers = {
-        "${(builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address}:8080" = {};
-      };
-      consul.services.ways-test-service = {
-        unit = "consul";
-        mode = "external";
-        definition = {
-          name = "ways-test-service";
-          address = (builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address;
-          port = 8080;
-        };
-      };
-    }))
-  ];
-
-  testScript = ''
-    import json
-    nodeNames = json.loads('${builtins.toJSON cluster.config.services.ways.nodes.host}')
-    nodes = [ n for n in machines if n.name in nodeNames ]
-
-    start_all()
-    nowhere.wait_for_unit("multi-user.target")
-    for node in nodes:
-      node.wait_for_unit("multi-user.target")
-
-    with subtest("single-target service"):
-      nowhere.succeed("curl -f https://ways-test-simple.${domain}")
-
-    with subtest("consul-managed service"):
-      nowhere.succeed("curl -f https://ways-test-consul.${domain}")
-  '';
-}
--- a/cluster/services/wireguard/test.nix
+++ b/cluster/services/wireguard/test.nix
@ -1,8 +1,6 @@
 { cluster, lib, ... }:

 {
-  defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
-
  testScript = ''
    start_all()
    ${lib.pipe cluster.config.services.wireguard.nodes.mesh [
--- a/flake.lock
+++ b/flake.lock
@ -40,11 +40,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1724226964,
-        "narHash": "sha256-cltFh4su2vcFidxKp7LuEgX3ZGLfPy0DCdrQZ/QTe68=",
+        "lastModified": 1723558887,
+        "narHash": "sha256-FBaRmikNnCX+HQJXAfNJyiOl3sM1NrlxvcFwQDqwCyk=",
        "owner": "zhaofengli",
        "repo": "attic",
-        "rev": "6d9aeaef0a067d664cb11bb7704f7ec373d47fb2",
+        "rev": "26b9417bde03edc6280d1f7ce709cd619cdb72d4",
        "type": "github"
      },
      "original": {
@ -76,11 +76,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1722960479,
-        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "lastModified": 1717025063,
+        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
        "type": "github"
      },
      "original": {
@ -504,11 +504,11 @@
    },
    "nixpkgs_4": {
      "locked": {
-        "lastModified": 1724221631,
-        "narHash": "sha256-zf3gNf0nX2yOb++h4jW9l4iG8R/LfazWZD+KhmW6fcc=",
+        "lastModified": 1723720832,
+        "narHash": "sha256-zoWgUEjVs7mmwY3L0aOrxdJBdACU2/2mMeJS6XElFAo=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "ac2df85f4d5c580786c7b4db031c199554152681",
+        "rev": "c46d43c625f161d3bd22052a5f8f9df703d81be8",
        "type": "github"
      },
      "original": {
--- a/modules/external-storage/default.nix
+++ b/modules/external-storage/default.nix
@ -8,8 +8,6 @@ let
  cfgAge = config.age;

  create = lib.flip lib.mapAttrs';
-
-  createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f;
 in

 {
@ -22,17 +20,12 @@ in
      fileSystems = lib.mkOption {
        description = "S3QL-based filesystems on top of CIFS mountpoints.";
        default = {};
-        type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let
-          authFile = if config.locksmithSecret != null then
-            "/run/locksmith/${config.locksmithSecret}"
-          else
-            cfgAge.secrets."storageAuth-${name}".path;
-        in {
+        type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: {
          imports = [ ./filesystem-type.nix ];
          backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}";
          commonArgs = [
            "--cachedir" config.cacheDir
-            "--authfile" authFile
+            "--authfile" cfgAge.secrets."storageAuth-${name}".path
          ] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]);
        }));
      };
@ -64,14 +57,9 @@ in

    age.secrets = lib.mkMerge [
      (create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; }))
-      (createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
+      (create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
    ];

-    services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: {
-      name = fs.unitName;
-      value = [ fs.locksmithSecret ];
-    });
-
    fileSystems = create cfg.underlays (name: ul: {
      name = ul.mountpoint;
      value = {
@ -109,13 +97,7 @@ in
        value = let
          isUnderlay = fs.underlay != null;

-          backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
-
-          fsType = if isUnderlay then "local" else lib.head backendParts;
-
-          s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
-
-          s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
+          fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);

          localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
        in {
@ -138,12 +120,8 @@ in
            ExecStartPre = map lib.escapeShellArgs [
              [
                (let
-                  authFile = if fs.locksmithSecret != null then
-                    "/run/locksmith/${fs.locksmithSecret}"
-                  else
-                    cfgAge.secrets."storageAuth-${name}".path;
                  mkfsEncrypted = ''
-                    ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
+                    ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
                      | cut -d' ' -f2- \
                      | ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
                  '';
@ -154,11 +132,6 @@ in

                  detectFs = {
                    local = "test -e ${localBackendPath}/s3ql_metadata";
-                    s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
-                      export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
-                      export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
-                      ${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
-                    '';
                  }.${fsType} or null;
                in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
                  if ! ${detectFs}; then
--- a/modules/external-storage/filesystem-type.nix
+++ b/modules/external-storage/filesystem-type.nix
@ -22,10 +22,6 @@ with lib;
    authFile = mkOption {
      type = types.path;
    };
-    locksmithSecret = mkOption {
-      type = with types; nullOr str;
-      default = null;
-    };
    cacheDir = mkOption {
      type = types.path;
      default = "/var/cache/remote-storage/${name}";
--- a/packages/checks/default.nix
+++ b/packages/checks/default.nix
@ -15,6 +15,12 @@ in
        inherit (config) cluster;
      };

+      garage = pkgs.callPackage ./garage.nix {
+        inherit (self'.packages) garage consul;
+        inherit (self) nixosModules;
+        inherit (config) cluster;
+      };
+
      ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
        inherit (self) nixosModules;
        previous = timeMachine.preUnstable;
--- a/packages/checks/garage.nix
+++ b/packages/checks/garage.nix
@ -0,0 +1,155 @@
+{ testers, nixosModules, cluster, garage, consul }:
+
+testers.runNixOSTest {
+  name = "garage";
+
+  imports = [
+    ./modules/consul.nix
+  ];
+
+  extraBaseModules.services.consul.package = consul;
+
+  nodes = let
+    common = { config, lib, ... }: let
+      inherit (config.networking) hostName primaryIPAddress;
+    in {
+      imports = lib.flatten [
+        ./modules/nixos/age-dummy-secrets
+        ./modules/nixos/age-dummy-secrets/options.nix
+        nixosModules.ascensions
+        nixosModules.systemd-extras
+        nixosModules.consul-distributed-services
+        nixosModules.port-magic
+        cluster.config.services.storage.nixos.garage
+        cluster.config.services.storage.nixos.garageInternal
+        cluster.config.services.consul.nixos.ready
+      ];
+      options.services.locksmith.providers = lib.mkOption {
+        type = lib.types.raw;
+      };
+      config = {
+        links.consulAgent = {
+          protocol = "http";
+          hostname = "consul";
+          port = 8500;
+        };
+        _module.args = {
+          depot.packages = { inherit garage; };
+          cluster.config = {
+            hostLinks.${hostName} = {
+              garageRpc.tuple = "${primaryIPAddress}:3901";
+              garageS3.tuple = "${primaryIPAddress}:8080";
+              garageWeb.tuple = "${primaryIPAddress}:8081";
+            };
+            links.garageWeb.hostname = "web.garage.example.com";
+            vars.meshNet.cidr = "192.168.0.0/16";
+          };
+        };
+        environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+        networking.firewall.allowedTCPPorts = [ 3901 8080 ];
+        services.garage = {
+          layout.initial = lib.mkOverride 51 {
+            garage1 = { zone = "dc1"; capacity = 1000; };
+            garage2 = { zone = "dc1"; capacity = 1000; };
+            garage3 = { zone = "dc1"; capacity = 1000; };
+          };
+        };
+        system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
+        specialisation.modifiedLayout = {
+          inheritParentConfig = true;
+          configuration = {
+            services.garage = {
+              layout.initial = lib.mkForce {
+                garage1 = { zone = "dc1"; capacity = 2000; };
+                garage2 = { zone = "dc1"; capacity = 1000; };
+                garage3 = { zone = "dc1"; capacity = 1000; };
+              };
+              keys.testKey.allow.createBucket = true;
+              buckets = {
+                bucket1 = {
+                  allow.testKey = [ "read" "write" ];
+                  quotas = {
+                    maxObjects = 300;
+                    maxSize = 400 * 1024 * 1024;
+                  };
+                };
+                bucket2 = {
+                  allow.testKey = [ "read" ];
+                };
+              };
+            };
+            system.ascensions.garage-layout.incantations = lib.mkForce (i: [
+              (i.runGarage ''
+                garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
+                garage layout apply --version 2
+              '')
+            ]);
+          };
+        };
+      };
+    };
+  in {
+    garage1.imports = [ common ];
+    garage2.imports = [ common ];
+    garage3.imports = [ common ];
+  };
+
+  testScript = { nodes, ... }: /*python*/ ''
+    nodes = [garage1, garage2, garage3]
+
+    start_all()
+
+    with subtest("should bootstrap new cluster"):
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+    with subtest("should apply new layout with ascension"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
+
+      for node in nodes:
+        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should apply new layout from scratch"):
+      for node in nodes:
+        node.systemctl("stop garage.service")
+        node.succeed("rm -rf /var/lib/garage-metadata")
+
+      for node in nodes:
+        node.systemctl("start garage.service")
+
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should create specified buckets and keys"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
+      garage1.succeed("garage key list | grep testKey")
+      garage1.succeed("garage bucket list | grep bucket1")
+      garage1.succeed("garage bucket list | grep bucket2")
+
+    with subtest("should delete unspecified buckets and keys"):
+      garage1.succeed("garage bucket create unwantedbucket")
+      garage1.succeed("garage key new --name unwantedkey")
+      garage1.succeed("systemctl restart garage-apply.service")
+
+      garage1.fail("garage key list | grep unwantedkey")
+      garage1.fail("garage bucket list | grep unwantedbucket")
+  '';
+}
--- a/packages/checks/s3ql-upgrade.nix
+++ b/packages/checks/s3ql-upgrade.nix
@ -10,9 +10,6 @@ testers.runNixOSTest {
      nixosModules.systemd-extras
      ./modules/nixos/age-dummy-secrets
      ./modules/nixos/age-dummy-secrets/options.nix
-      {
-        options.services.locksmith = lib.mkSinkUndeclaredOptions { };
-      }
    ];

    _module.args.depot.packages = { inherit (previous.packages.${system}) s3ql; };
--- a/packages/patched-inputs.nix
+++ b/packages/patched-inputs.nix
@ -11,6 +11,8 @@
      nix-super = packages.nix-super.nix;

      agenix = packages.agenix.agenix.override { nix = nix-super; };
+
+      hci = packages.hercules-ci-agent.hercules-ci-cli;
    };
  };
 }
--- a/packages/projects.nix
+++ b/packages/projects.nix
@ -59,6 +59,7 @@
        tools = with flakePkgs; [
          agenix
          graf
+          hci
          npins
          pin
          void
--- a/packages/system-filter.nix
+++ b/packages/system-filter.nix
@ -1,5 +1,6 @@
 {
  packages = {
+    hci = [ "x86_64-linux" ];
    hydra = [ "x86_64-linux" ];
    jellyfin = [ "x86_64-linux" ];
    keycloak = [ "x86_64-linux" ];