flake.lock: Update

Flake lock file updates:

• Updated input 'agenix':
    'github:ryantm/agenix/3f1dae074a12feb7327b4bf43cbac0d124488bb7?narHash=sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU%3D' (2024-07-30)
  → 'github:ryantm/agenix/f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41?narHash=sha256-b%2Buqzj%2BWa6xgMS9aNbX4I%2BsXeb5biPDi39VgvSFqFvU%3D' (2024-08-10)
• Updated input 'attic':
    'github:zhaofengli/attic/e127acbf9a71ebc0c26bc8e28346822e0a6e16ba?narHash=sha256-GJIz4M5HDB948Ex/8cPvbkrNzl/eKUE7/c21JBu4lb8%3D' (2024-08-01)
  → 'github:zhaofengli/attic/6d9aeaef0a067d664cb11bb7704f7ec373d47fb2?narHash=sha256-cltFh4su2vcFidxKp7LuEgX3ZGLfPy0DCdrQZ/QTe68%3D' (2024-08-21)
• Updated input 'attic/crane':
    'github:ipetkov/crane/480dff0be03dac0e51a8dfc26e882b0d123a450e?narHash=sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8%3D' (2024-05-29)
  → 'github:ipetkov/crane/4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4?narHash=sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y%3D' (2024-08-06)
• Updated input 'hercules-ci-agent':
    'github:hercules-ci/hercules-ci-agent/2e10fb21fc2e07edf40763b73443e5934bd40947?narHash=sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4%3D' (2024-07-05)
  → 'github:hercules-ci/hercules-ci-agent/c303cc8e437c0fd26b9452472e7df5aa374e9177?narHash=sha256-/Vdg5ZKtP71ZEKVV6JXlrOEu0CM2Flcs%2BnwDmWRzgjQ%3D' (2024-08-15)
• Updated input 'nar-serve':
    'github:numtide/nar-serve/9d0eff868d328fe67c60c26c8ba50e0b9d8de867?narHash=sha256-8QuMS00EutmqzAIPxyJEPxM8EHiWlSKs6E2Htoh3Kes%3D' (2024-07-31)
  → 'github:numtide/nar-serve/846cb9d55aee9af7c7a1eaf7016f2c5b9e01cc6e?narHash=sha256-i%2B9i1D5zEz2c3o5RuH%2BX/jDRmA12vgU8UxxE0/TPvtE%3D' (2024-08-15)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/f2d6c7123138044e0c68902268bd8f37dd7e2fa7?narHash=sha256-g4L%2BI8rDl7RQy5x8XcEMqNO49LFhrHTzVBqXtG2%2BFGo%3D' (2024-08-01)
  → 'github:NixOS/nixpkgs/ac2df85f4d5c580786c7b4db031c199554152681?narHash=sha256-zf3gNf0nX2yOb%2B%2Bh4jW9l4iG8R/LfazWZD%2BKhmW6fcc%3D' (2024-08-21)

cluster/secrets/forge-dbCredentials.age

@@ -1,13 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
-JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
--> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
-2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
--> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
-bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
--> Ovax-grease ^1$]}H G4 FpDF XKHkj{
-IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
---- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
-ÜA³×oÈð:±­‹`ÜVd±å(Kät:fk¼’}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69i›l`ÛÆJKwAè8­y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
-üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ý’wÊ
-ÆÜßÌZî!^þRˆéÿv­¾…ïk‹Êp»ÛPÌ)ý̆ÍpÓV5²F΄ÆÚÙÚÞhBÇ»ßb#Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè‹}€ß	ƒ¬E}~ZHûjmyq{òxŠ–Éôß<C3B4>"”éÀ´C#šójÿÐ.ò§yÔ£¸v¦
<0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që

cluster/services/attic/default.nix

@@ -33,7 +33,7 @@
     };
   };
 
-  garage = {
+  garage = config.lib.forService "attic" {
     keys.attic.locksmith = {
       nodes = config.services.attic.nodes.server;
       owner = "atticd";
@@ -48,14 +48,16 @@
     serverAddrs = map
       (node: depot.hours.${node}.interfaces.primary.addrPublic)
       config.services.attic.nodes.server;
-  in {
+  in config.lib.forService "attic" {
     cache.target = serverAddrs;
   };
 
-  ways.cache-api = {
-    consulService = "atticd";
-    extras.extraConfig = ''
-      client_max_body_size 4G;
-    '';
+  ways = config.lib.forService "attic" {
+    cache-api = {
+      consulService = "atticd";
+      extras.extraConfig = ''
+        client_max_body_size 4G;
+      '';
+    };
   };
 }

cluster/services/consul/test.nix

@@ -1,4 +1,8 @@
+{ lib, ... }:
+
 {
+  defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
+
   testScript = ''
     import json
 

cluster/services/forge/default.nix

@@ -13,25 +13,36 @@
         nodes = server;
         owner = "forgejo";
       };
-      dbCredentials.nodes = server;
     };
   };
 
-  ways.forge.target = let
+  ways = let
     host = builtins.head config.services.forge.nodes.server;
-  in config.hostLinks.${host}.forge.url;
+  in config.lib.forService "forge" {
+    forge.target = config.hostLinks.${host}.forge.url;
+  };
 
-  garage = {
+  patroni = config.lib.forService "forge" {
+    databases.forge = {};
+    users.forge.locksmith = {
+      nodes = config.services.forge.nodes.server;
+      format = "raw";
+    };
+  };
+
+  garage = config.lib.forService "forge" {
     keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
     buckets.forgejo.allow.forgejo = [ "read" "write" ];
   };
 
-  monitoring.blackbox.targets.forge = {
+  monitoring.blackbox.targets.forge = config.lib.forService "forge" {
     address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
     module = "https2xx";
   };
 
-  dns.records."ssh.forge".target = map
-    (node: depot.hours.${node}.interfaces.primary.addrPublic)
-    config.services.forge.nodes.server;
+  dns.records = config.lib.forService "forge" {
+    "ssh.forge".target = map
+      (node: depot.hours.${node}.interfaces.primary.addrPublic)
+      config.services.forge.nodes.server;
+  };
 }

cluster/services/forge/server.nix

@@ -26,6 +26,7 @@ in
   services.locksmith.waitForSecrets.forgejo = [
     "garage-forgejo-id"
     "garage-forgejo-secret"
+    "patroni-forge"
   ];
 
   services.forgejo = {
@@ -39,7 +40,7 @@ in
       inherit (patroni) port;
       name = "forge";
       user = "forge";
-      passwordFile = secrets.dbCredentials.path;
+      passwordFile = "/run/locksmith/patroni-forge";
     };
     settings = {
       DEFAULT = {

cluster/services/hercules-ci-multi-agent/default.nix

@@ -62,7 +62,7 @@
       lib.unique
       (map (x: "hci-agent-${x}"))
     ];
-  in {
+  in config.lib.forService "hercules-ci-multi-agent" {
     keys = lib.genAttrs hciAgentKeys (lib.const {});
     buckets.nix-store = {
       allow = lib.genAttrs hciAgentKeys (lib.const [ "read" "write" ]);

cluster/services/monitoring/default.nix

@@ -72,7 +72,7 @@ in
     };
   };
 
-  garage = {
+  garage = config.lib.forService "monitoring" {
     keys = {
       loki-ingest.locksmith = {
         nodes = config.services.monitoring.nodes.logging;
@@ -93,7 +93,7 @@ in
     };
   };
 
-  ways = {
+  ways = config.lib.forService "monitoring" {
     monitoring = {
       consulService = "grafana";
       extras.locations."/".proxyWebsockets = true;

cluster/services/monitoring/provisioning/objects/dashboards/dashboard-dc7545ef-3180-4a5e-a289-1e64571ebb87.json

@@ -19,10 +19,23 @@
     "editable": true,
     "fiscalYearStartMonth": 0,
     "graphTooltip": 0,
-    "id": 16,
+    "id": 1210,
     "links": [],
     "liveNow": false,
     "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 7,
+        "panels": [],
+        "title": "Replication",
+        "type": "row"
+      },
       {
         "datasource": {
           "type": "prometheus",
@@ -58,7 +71,7 @@
           "h": 8,
           "w": 12,
           "x": 0,
-          "y": 0
+          "y": 1
         },
         "id": 2,
         "options": {
@@ -66,6 +79,7 @@
           "graphMode": "area",
           "justifyMode": "auto",
           "orientation": "auto",
+          "percentChangeColorMode": "standard",
           "reduceOptions": {
             "calcs": [
               "lastNotNull"
@@ -73,9 +87,11 @@
             "fields": "",
             "values": false
           },
-          "textMode": "auto"
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
         },
-        "pluginVersion": "9.5.1",
+        "pluginVersion": "11.1.3",
         "targets": [
           {
             "datasource": {
@@ -138,7 +154,7 @@
           "h": 8,
           "w": 4,
           "x": 12,
-          "y": 0
+          "y": 1
         },
         "id": 3,
         "options": {
@@ -146,14 +162,17 @@
           "graphMode": "area",
           "justifyMode": "auto",
           "orientation": "auto",
+          "percentChangeColorMode": "standard",
           "reduceOptions": {
             "calcs": [],
             "fields": "/^instance$/",
             "values": true
           },
-          "textMode": "auto"
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
         },
-        "pluginVersion": "9.5.1",
+        "pluginVersion": "11.1.3",
         "targets": [
           {
             "datasource": {
@@ -198,7 +217,7 @@
           "h": 8,
           "w": 8,
           "x": 16,
-          "y": 0
+          "y": 1
         },
         "id": 5,
         "options": {
@@ -254,6 +273,7 @@
               "mode": "palette-classic"
             },
             "custom": {
+              "axisBorderShow": false,
               "axisCenteredZero": false,
               "axisColorMode": "text",
               "axisLabel": "",
@@ -267,6 +287,7 @@
                 "tooltip": false,
                 "viz": false
               },
+              "insertNulls": false,
               "lineInterpolation": "smooth",
               "lineStyle": {
                 "fill": "solid"
@@ -304,7 +325,7 @@
           "h": 13,
           "w": 12,
           "x": 0,
-          "y": 8
+          "y": 9
         },
         "id": 1,
         "options": {
@@ -348,6 +369,7 @@
               "mode": "palette-classic"
             },
             "custom": {
+              "axisBorderShow": false,
               "axisCenteredZero": false,
               "axisColorMode": "text",
               "axisLabel": "",
@@ -361,6 +383,7 @@
                 "tooltip": false,
                 "viz": false
               },
+              "insertNulls": false,
               "lineInterpolation": "smooth",
               "lineWidth": 1,
               "pointSize": 5,
@@ -398,7 +421,7 @@
           "h": 13,
           "w": 12,
           "x": 12,
-          "y": 8
+          "y": 9
         },
         "id": 4,
         "options": {
@@ -428,24 +451,326 @@
         ],
         "title": "Activity",
         "type": "timeseries"
+      },
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 22
+        },
+        "id": 8,
+        "panels": [],
+        "title": "Storage",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              }
+            },
+            "mappings": [],
+            "unit": "bytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 15,
+          "w": 8,
+          "x": 0,
+          "y": 23
+        },
+        "id": 6,
+        "options": {
+          "displayLabels": [
+            "percent"
+          ],
+          "legend": {
+            "calcs": [],
+            "displayMode": "table",
+            "placement": "right",
+            "showLegend": true,
+            "sortBy": "Value",
+            "sortDesc": true,
+            "values": [
+              "percent",
+              "value"
+            ]
+          },
+          "pieType": "donut",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "pluginVersion": "11.1.3",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "disableTextWrap": false,
+            "editorMode": "builder",
+            "exemplar": false,
+            "expr": "max by(datname) (sum by(datname, instance) (pg_database_size_bytes))",
+            "fullMetaSearch": false,
+            "includeNullMetadata": false,
+            "instant": true,
+            "legendFormat": "__auto",
+            "range": false,
+            "refId": "A",
+            "useBackend": false
+          }
+        ],
+        "title": "Database Size",
+        "type": "piechart"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 25,
+              "gradientMode": "opacity",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "bytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 15,
+          "w": 5,
+          "x": 8,
+          "y": 23
+        },
+        "id": 9,
+        "options": {
+          "legend": {
+            "calcs": [
+              "lastNotNull",
+              "max"
+            ],
+            "displayMode": "table",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "pluginVersion": "11.1.3",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "disableTextWrap": false,
+            "editorMode": "builder",
+            "exemplar": false,
+            "expr": "pg_wal_size_bytes",
+            "format": "time_series",
+            "fullMetaSearch": false,
+            "includeNullMetadata": true,
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A",
+            "useBackend": false
+          }
+        ],
+        "title": "WAL Size",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds",
+              "seriesBy": "last"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 25,
+              "gradientMode": "opacity",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineStyle": {
+                "fill": "solid"
+              },
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "decimals": 3,
+            "fieldMinMax": false,
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                }
+              ]
+            },
+            "unit": "bytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 15,
+          "w": 11,
+          "x": 13,
+          "y": 23
+        },
+        "id": 10,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "disableTextWrap": false,
+            "editorMode": "builder",
+            "exemplar": false,
+            "expr": "max(sum by(instance) (pg_database_size_bytes))",
+            "fullMetaSearch": false,
+            "includeNullMetadata": false,
+            "instant": false,
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A",
+            "useBackend": false
+          }
+        ],
+        "title": "Total DB Size",
+        "type": "timeseries"
       }
     ],
     "refresh": "5m",
-    "schemaVersion": 38,
-    "style": "dark",
+    "schemaVersion": 39,
     "tags": [],
     "templating": {
       "list": []
     },
     "time": {
-      "from": "now-1h",
+      "from": "now-24h",
       "to": "now"
     },
     "timepicker": {},
     "timezone": "",
     "title": "PostgreSQL HA",
     "uid": "dc7545ef-3180-4a5e-a289-1e64571ebb87",
-    "version": 7,
+    "version": 2,
     "weekStart": ""
   },
   "folderId": 0,

cluster/services/storage/default.nix

@@ -7,6 +7,8 @@ in
 {
   imports = [
     ./options.nix
+    ./incandescence.nix
+    ./simulacrum/test-data.nix
   ];
 
   services.storage = {
@@ -30,11 +32,15 @@ in
       heresy = [
         ./heresy.nix
         ./s3ql-upgrades.nix
+      ] ++ lib.optionals config.simulacrum [
+        ./simulacrum/snakeoil-heresy-passphrase.nix
       ];
       garage = [
         ./garage.nix
         ./garage-options.nix
         ./garage-layout.nix
+      ] ++ lib.optionals config.simulacrum [
+        ./simulacrum/snakeoil-rpc-secret.nix
       ];
       garageConfig = [
         ./garage-gateway.nix
@@ -48,6 +54,11 @@ in
       garageInternal = [ ./garage-internal.nix ];
       garageExternal = [ ./garage-external.nix ];
     };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
+      settings = ./simulacrum/test.nix;
+    };
   };
 
   links = {
@@ -86,7 +97,10 @@ in
   };
 
   garage = {
-    keys.storage-prophet = {};
+    keys.storage-prophet.locksmith = {
+      nodes = [ "prophet" ];
+      format = "s3ql";
+    };
     buckets.storage-prophet = {
       allow.storage-prophet = [ "read" "write" ];
     };

cluster/services/storage/external.nix

@@ -8,7 +8,7 @@ in
   services.external-storage = {
     fileSystems.external = {
       mountpoint = "/srv/storage";
-      authFile = ./secrets/external-storage-auth-${hostName}.age;
+      locksmithSecret = "garage-storage-${hostName}";
       backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
       backendOptions = [ "disable-expect100" ];
     };

cluster/services/storage/garage-options.nix

@@ -26,62 +26,6 @@ let
         sleep 1
       done
     }
-    # FIXME: returns bogus empty string when one of the lists is empty
-    diffAdded() {
-      comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
-    }
-    diffRemoved() {
-      comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
-    }
-    # FIXME: this does not handle list items with spaces
-    listKeys() {
-      garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
-    }
-    ensureKeys() {
-      old="$(listKeys)"
-      if [[ -z "$1" ]]; then
-        for key in $old; do
-          garage key delete --yes "$key"
-        done
-      elif [[ -z "$old" ]]; then
-        for key in $1; do
-          # don't print secret key
-          garage key new --name "$key" >/dev/null
-          echo Key "$key" was created.
-        done
-      else
-        diffAdded "$old" "$1" | while read key; do
-          # don't print secret key
-          garage key new --name "$key" >/dev/null
-          echo Key "$key" was created.
-        done
-        diffRemoved "$old" "$1" | while read key; do
-          garage key delete --yes "$key"
-        done
-      fi
-    }
-    listBuckets() {
-      garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
-    }
-    ensureBuckets() {
-      old="$(listBuckets)"
-      if [[ -z "$1" ]]; then
-        for bucket in $old; do
-          garage bucket delete --yes "$bucket"
-        done
-      elif [[ -z "$old" ]]; then
-        for bucket in $1; do
-          garage bucket create "$bucket"
-        done
-      else
-        diffAdded "$old" "$1" | while read bucket; do
-          garage bucket create "$bucket"
-        done
-        diffRemoved "$old" "$1" | while read bucket; do
-          garage bucket delete --yes "$bucket"
-        done
-      fi
-    }
   '';
 in
 
@@ -118,7 +62,7 @@ in
             };
             format = mkOption {
               description = "Locksmith secret format.";
-              type = enum [ "files" "aws" "envFile" ];
+              type = enum [ "files" "aws" "envFile" "s3ql" ];
               default = "files";
             };
             owner = mkOption {
@@ -203,9 +147,7 @@ in
           garage layout apply --version 1
         '';
       };
-      garage-apply = {
-        distributed.enable = true;
-        wantedBy = [ "garage.service" "multi-user.target" ];
+      garage-ready = {
         wants = [ "garage.service" ];
         after = [ "garage.service" "garage-layout-init.service" ];
         path = [ config.services.garage.package ];
@@ -219,54 +161,78 @@ in
         script = ''
           source ${garageShellLibrary}
           waitForGarageOperational
-
-          ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
-          ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
-
-          # key permissions
-          ${lib.pipe cfg.keys [
-            (lib.mapAttrsToList (key: kCfg: ''
-              garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
-            ''))
-            (lib.concatStringsSep "\n")
-          ]}
-
-          # bucket permissions
-          ${lib.pipe cfg.buckets [
-            (lib.mapAttrsToList (bucket: bCfg:
-              lib.mapAttrsToList (key: perms: ''
-                garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
-                garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
-              '') bCfg.allow
-            ))
-            lib.flatten
-            (lib.concatStringsSep "\n")
-          ]}
-
-          # bucket quotas
-          ${lib.pipe cfg.buckets [
-            (lib.mapAttrsToList (bucket: bCfg: ''
-              garage bucket set-quotas '${bucket}' \
-                --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
-                --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
-            ''))
-            (lib.concatStringsSep "\n")
-          ]}
-
-          # bucket website access
-          ${lib.pipe cfg.buckets [
-            (lib.mapAttrsToList (bucket: bCfg: ''
-              garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
-            ''))
-            (lib.concatStringsSep "\n")
-          ]}
         '';
       };
     };
 
+    services.incandescence.providers.garage = {
+      locksmith = true;
+      wantedBy = [ "garage.service" "multi-user.target" ];
+      partOf = [ "garage.service" ];
+      wants = [ "garage-ready.service" ];
+      after = [ "garage-ready.service" ];
+
+      packages = [
+        config.services.garage.package
+      ];
+      formulae = {
+        key = {
+          destroyAfterDays = 0;
+          create = key: ''
+            if [[ "$(garage key info ${lib.escapeShellArg key} 2>&1 >/dev/null)" == "Error: 0 matching keys" ]]; then
+              # don't print secret key
+              garage key new --name ${lib.escapeShellArg key} >/dev/null
+              echo Key ${lib.escapeShellArg key} was created.
+            else
+              echo "Key already exists, assuming ownership"
+            fi
+          '';
+          destroy = ''
+            garage key delete --yes "$OBJECT"
+          '';
+          change = key: let
+            kCfg = cfg.keys.${key};
+          in ''
+            garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
+          '';
+        };
+        bucket = {
+          deps = [ "key" ];
+          destroyAfterDays = 30;
+          create = bucket: ''
+            if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == "Error: Bucket not found" ]]; then
+              garage bucket create ${lib.escapeShellArg bucket}
+            else
+              echo "Bucket already exists, assuming ownership"
+            fi
+          '';
+          destroy = ''
+            garage bucket delete --yes "$OBJECT"
+          '';
+          change = bucket: let
+            bCfg = cfg.buckets.${bucket};
+          in ''
+            # permissions
+            ${lib.concatStringsSep "\n" (lib.flatten (
+              lib.mapAttrsToList (key: perms: ''
+                garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
+                garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
+              '') bCfg.allow
+            ))}
+
+            # quotas
+            garage bucket set-quotas ${lib.escapeShellArg bucket} \
+              --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
+              --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
+
+            # website access
+            garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
+          '';
+        };
+      };
+    };
+
     services.locksmith.providers.garage = {
-      wantedBy = [ "garage-apply.service" ];
-      after = [ "garage-apply.service" ];
       secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
         common = {
           inherit (kCfg.locksmith) mode owner group nodes;
@@ -291,6 +257,12 @@ in
             AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
             AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
           '';
+          s3ql = ''
+            [s3c]
+            storage-url: s3c4://
+            backend-login: @@GARAGE_KEY_ID@@
+            backend-password: @@GARAGE_SECRET_KEY@@
+          '';
         }.${kCfg.locksmith.format};
       in {
         ${key} = common // {

cluster/services/storage/garage.nix

@@ -63,6 +63,8 @@ in
   };
 
   systemd.services.garage = {
+    requires = [ "consul-ready.service" ];
+    after = [ "consul-ready.service" ];
     unitConfig = {
       RequiresMountsFor = [ cfg.settings.data_dir ];
     };

cluster/services/storage/heresy.nix

@@ -11,6 +11,7 @@
       unitDescription = "Heresy Filesystem";
       authFile = ./secrets/heresy-encryption-key.age;
       underlay = "heresy";
+      encrypt = true;
     };
   };
 }

cluster/services/storage/incandescence.nix

@@ -0,0 +1,10 @@
+{ config, lib, ... }:
+
+{
+  incandescence.providers.garage = {
+    objects = {
+      key = lib.attrNames config.garage.keys;
+      bucket = lib.attrNames config.garage.buckets;
+    };
+  };
+}

cluster/services/storage/secrets/external-storage-auth-prophet.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
-2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
--> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
-0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
--> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
-LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
---- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
-™ªg¬NÑ8´¨\K!p
«ï…7ù¶›käõ¯#ŒÏu›µ*{}Tþ0·|@ÉÿàE>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†‡4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
-µÿÂ9gÏi"Q
–ÿ
-™›Ù®à

cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix

@@ -0,0 +1,8 @@
+{
+  environment.etc."dummy-secrets/storageAuth-heresy".text = ''
+    [local]
+    storage-url: local://
+    fs-passphrase: simulacrum
+  '';
+}
+

cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix

@@ -0,0 +1,3 @@
+{
+  environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+}

cluster/services/storage/simulacrum/test-data.nix

@@ -0,0 +1,8 @@
+{ config, lib, ... }:
+
+{
+  garage = lib.mkIf config.simulacrum {
+    keys.testkey = {};
+    buckets.testbucket.allow.testKey = [ "read" "write" ];
+  };
+}

cluster/services/storage/simulacrum/test.nix

@@ -0,0 +1,105 @@
+{ cluster, lib, ... }:
+
+let
+  inherit (cluster.config.services.storage) nodes;
+
+  firstGarageNode = lib.elemAt nodes.garage 0;
+in
+
+{
+  nodes = lib.genAttrs nodes.garage (node: {
+    services.garage = {
+      layout.initial = lib.genAttrs nodes.garage (_: {
+        capacity = lib.mkOverride 51 1000;
+      });
+    };
+    specialisation.modifiedLayout = {
+      inheritParentConfig = true;
+      configuration = {
+        services.garage = {
+          layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
+        };
+        system.ascensions.garage-layout.incantations = lib.mkForce (i: [
+          (i.runGarage ''
+            garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
+            garage layout apply --version 2
+          '')
+        ]);
+      };
+    };
+  });
+
+  testScript = ''
+    import json
+    nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
+    garage1 = nodes[0]
+
+    start_all()
+
+    with subtest("should bootstrap new cluster"):
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+    with subtest("should apply new layout with ascension"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
+
+      for node in nodes:
+        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
+    addr = consulConfig["addresses"]["http"]
+    port = consulConfig["ports"]["http"]
+    setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
+    with subtest("should apply new layout from scratch"):
+      for node in nodes:
+        node.systemctl("stop garage.service")
+        node.succeed("rm -rf /var/lib/garage-metadata")
+      garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
+
+      for node in nodes:
+        node.systemctl("start garage.service")
+
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should create specified buckets and keys"):
+      for node in nodes:
+        node.wait_for_unit("incandescence-garage.target")
+      garage1.succeed("garage key list | grep testkey")
+      garage1.succeed("garage bucket list | grep testbucket")
+
+    with subtest("should delete unspecified keys"):
+      garage1.succeed("garage bucket create unwantedbucket")
+      garage1.succeed("garage key new --name unwantedkey")
+      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
+      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
+      garage1.succeed("systemctl restart garage.service")
+      garage1.wait_for_unit("incandescence-garage.target")
+      garage1.fail("garage key list | grep unwantedkey")
+      garage1.succeed("garage bucket list | grep unwantedbucket")
+
+    with subtest("should delete unspecified buckets after grace period"):
+      garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
+      garage1.succeed("systemctl restart garage.service")
+      garage1.wait_for_unit("incandescence-garage.target")
+      garage1.fail("garage bucket list | grep unwantedbucket")
+  '';
+}
+

cluster/services/ways/default.nix

@@ -3,11 +3,17 @@
 {
   imports = [
     ./options
+    ./simulacrum/test-data.nix
   ];
 
   services.ways = {
     nodes.host = config.services.websites.nodes.host;
     nixos.host = ./host.nix;
+    simulacrum = {
+      enable = true;
+      deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
+      settings = ./simulacrum/test.nix;
+    };
   };
 
   dns.records = lib.mapAttrs'

cluster/services/ways/simulacrum/test-data.nix

@@ -0,0 +1,11 @@
+{ config, lib, ... }:
+{
+  ways = lib.mkIf config.simulacrum {
+    ways-test-simple = config.lib.forService "ways" {
+      target = "http://nowhere";
+    };
+    ways-test-consul = config.lib.forService "ways" {
+      consulService = "ways-test-service";
+    };
+  };
+}

cluster/services/ways/simulacrum/test.nix

@@ -0,0 +1,55 @@
+{ cluster, config, lib, ... }:
+
+let
+  inherit (cluster._module.specialArgs.depot.lib.meta) domain;
+in
+
+{
+  nodes = lib.mkMerge [
+    {
+      nowhere = { pkgs, ... }: {
+        networking.firewall.allowedTCPPorts = [ 8080 ];
+        systemd.services.ways-simple-service = let
+          webroot = pkgs.writeTextDir "example.txt" "hello world";
+        in {
+          wantedBy = [ "multi-user.target" ];
+          serviceConfig = {
+            ExecStart = "${pkgs.darkhttpd}/bin/darkhttpd ${webroot} --port 8080";
+            DynamicUser = true;
+          };
+        };
+      };
+    }
+    (lib.genAttrs cluster.config.services.ways.nodes.host (lib.const {
+      services.nginx.upstreams.nowhere.servers = {
+        "${(builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address}:8080" = {};
+      };
+      consul.services.ways-test-service = {
+        unit = "consul";
+        mode = "external";
+        definition = {
+          name = "ways-test-service";
+          address = (builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address;
+          port = 8080;
+        };
+      };
+    }))
+  ];
+
+  testScript = ''
+    import json
+    nodeNames = json.loads('${builtins.toJSON cluster.config.services.ways.nodes.host}')
+    nodes = [ n for n in machines if n.name in nodeNames ]
+
+    start_all()
+    nowhere.wait_for_unit("multi-user.target")
+    for node in nodes:
+      node.wait_for_unit("multi-user.target")
+
+    with subtest("single-target service"):
+      nowhere.succeed("curl -f https://ways-test-simple.${domain}")
+
+    with subtest("consul-managed service"):
+      nowhere.succeed("curl -f https://ways-test-consul.${domain}")
+  '';
+}

cluster/services/wireguard/test.nix

@@ -1,6 +1,8 @@
 { cluster, lib, ... }:
 
 {
+  defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
+
   testScript = ''
     start_all()
     ${lib.pipe cluster.config.services.wireguard.nodes.mesh [

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1722339003,
-        "narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
+        "lastModified": 1723293904,
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
         "type": "github"
       },
       "original": {
@@ -40,11 +40,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1722472866,
-        "narHash": "sha256-GJIz4M5HDB948Ex/8cPvbkrNzl/eKUE7/c21JBu4lb8=",
+        "lastModified": 1724226964,
+        "narHash": "sha256-cltFh4su2vcFidxKp7LuEgX3ZGLfPy0DCdrQZ/QTe68=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "e127acbf9a71ebc0c26bc8e28346822e0a6e16ba",
+        "rev": "6d9aeaef0a067d664cb11bb7704f7ec373d47fb2",
         "type": "github"
       },
       "original": {
@@ -76,11 +76,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717025063,
-        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
+        "lastModified": 1722960479,
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
         "type": "github"
       },
       "original": {
@@ -268,11 +268,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1720223941,
-        "narHash": "sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4=",
+        "lastModified": 1723736589,
+        "narHash": "sha256-/Vdg5ZKtP71ZEKVV6JXlrOEu0CM2Flcs+nwDmWRzgjQ=",
         "owner": "hercules-ci",
         "repo": "hercules-ci-agent",
-        "rev": "2e10fb21fc2e07edf40763b73443e5934bd40947",
+        "rev": "c303cc8e437c0fd26b9452472e7df5aa374e9177",
         "type": "github"
       },
       "original": {
@@ -399,11 +399,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1722409392,
-        "narHash": "sha256-8QuMS00EutmqzAIPxyJEPxM8EHiWlSKs6E2Htoh3Kes=",
+        "lastModified": 1723717449,
+        "narHash": "sha256-i+9i1D5zEz2c3o5RuH+X/jDRmA12vgU8UxxE0/TPvtE=",
         "owner": "numtide",
         "repo": "nar-serve",
-        "rev": "9d0eff868d328fe67c60c26c8ba50e0b9d8de867",
+        "rev": "846cb9d55aee9af7c7a1eaf7016f2c5b9e01cc6e",
         "type": "github"
       },
       "original": {
@@ -504,11 +504,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1722539632,
-        "narHash": "sha256-g4L+I8rDl7RQy5x8XcEMqNO49LFhrHTzVBqXtG2+FGo=",
+        "lastModified": 1724221631,
+        "narHash": "sha256-zf3gNf0nX2yOb++h4jW9l4iG8R/LfazWZD+KhmW6fcc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f2d6c7123138044e0c68902268bd8f37dd7e2fa7",
+        "rev": "ac2df85f4d5c580786c7b4db031c199554152681",
         "type": "github"
       },
       "original": {

modules/external-storage/default.nix

@@ -8,6 +8,8 @@ let
   cfgAge = config.age;
 
   create = lib.flip lib.mapAttrs';
+
+  createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f;
 in
 
 {
@@ -20,12 +22,17 @@ in
       fileSystems = lib.mkOption {
         description = "S3QL-based filesystems on top of CIFS mountpoints.";
         default = {};
-        type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: {
+        type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let
+          authFile = if config.locksmithSecret != null then
+            "/run/locksmith/${config.locksmithSecret}"
+          else
+            cfgAge.secrets."storageAuth-${name}".path;
+        in {
           imports = [ ./filesystem-type.nix ];
           backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}";
           commonArgs = [
             "--cachedir" config.cacheDir
-            "--authfile" cfgAge.secrets."storageAuth-${name}".path
+            "--authfile" authFile
           ] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]);
         }));
       };
@@ -57,9 +64,14 @@ in
 
     age.secrets = lib.mkMerge [
       (create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; }))
-      (create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
+      (createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
     ];
 
+    services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: {
+      name = fs.unitName;
+      value = [ fs.locksmithSecret ];
+    });
+
     fileSystems = create cfg.underlays (name: ul: {
       name = ul.mountpoint;
       value = {
@@ -97,7 +109,13 @@ in
         value = let
           isUnderlay = fs.underlay != null;
 
-          fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);
+          backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
+
+          fsType = if isUnderlay then "local" else lib.head backendParts;
+
+          s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
+
+          s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
 
           localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
         in {
@@ -120,8 +138,12 @@ in
             ExecStartPre = map lib.escapeShellArgs [
               [
                 (let
+                  authFile = if fs.locksmithSecret != null then
+                    "/run/locksmith/${fs.locksmithSecret}"
+                  else
+                    cfgAge.secrets."storageAuth-${name}".path;
                   mkfsEncrypted = ''
-                    ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
+                    ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
                       | cut -d' ' -f2- \
                       | ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
                   '';
@@ -132,6 +154,11 @@ in
 
                   detectFs = {
                     local = "test -e ${localBackendPath}/s3ql_metadata";
+                    s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
+                      export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
+                      export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
+                      ${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
+                    '';
                   }.${fsType} or null;
                 in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
                   if ! ${detectFs}; then

modules/external-storage/filesystem-type.nix

@@ -22,6 +22,10 @@ with lib;
     authFile = mkOption {
       type = types.path;
     };
+    locksmithSecret = mkOption {
+      type = with types; nullOr str;
+      default = null;
+    };
     cacheDir = mkOption {
       type = types.path;
       default = "/var/cache/remote-storage/${name}";

packages/checks/default.nix

@@ -15,12 +15,6 @@ in
         inherit (config) cluster;
       };
 
-      garage = pkgs.callPackage ./garage.nix {
-        inherit (self'.packages) garage consul;
-        inherit (self) nixosModules;
-        inherit (config) cluster;
-      };
-
       ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
         inherit (self) nixosModules;
         previous = timeMachine.preUnstable;

packages/checks/garage.nix

@@ -1,155 +0,0 @@
-{ testers, nixosModules, cluster, garage, consul }:
-
-testers.runNixOSTest {
-  name = "garage";
-
-  imports = [
-    ./modules/consul.nix
-  ];
-
-  extraBaseModules.services.consul.package = consul;
-
-  nodes = let
-    common = { config, lib, ... }: let
-      inherit (config.networking) hostName primaryIPAddress;
-    in {
-      imports = lib.flatten [
-        ./modules/nixos/age-dummy-secrets
-        ./modules/nixos/age-dummy-secrets/options.nix
-        nixosModules.ascensions
-        nixosModules.systemd-extras
-        nixosModules.consul-distributed-services
-        nixosModules.port-magic
-        cluster.config.services.storage.nixos.garage
-        cluster.config.services.storage.nixos.garageInternal
-        cluster.config.services.consul.nixos.ready
-      ];
-      options.services.locksmith.providers = lib.mkOption {
-        type = lib.types.raw;
-      };
-      config = {
-        links.consulAgent = {
-          protocol = "http";
-          hostname = "consul";
-          port = 8500;
-        };
-        _module.args = {
-          depot.packages = { inherit garage; };
-          cluster.config = {
-            hostLinks.${hostName} = {
-              garageRpc.tuple = "${primaryIPAddress}:3901";
-              garageS3.tuple = "${primaryIPAddress}:8080";
-              garageWeb.tuple = "${primaryIPAddress}:8081";
-            };
-            links.garageWeb.hostname = "web.garage.example.com";
-            vars.meshNet.cidr = "192.168.0.0/16";
-          };
-        };
-        environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-        networking.firewall.allowedTCPPorts = [ 3901 8080 ];
-        services.garage = {
-          layout.initial = lib.mkOverride 51 {
-            garage1 = { zone = "dc1"; capacity = 1000; };
-            garage2 = { zone = "dc1"; capacity = 1000; };
-            garage3 = { zone = "dc1"; capacity = 1000; };
-          };
-        };
-        system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
-        specialisation.modifiedLayout = {
-          inheritParentConfig = true;
-          configuration = {
-            services.garage = {
-              layout.initial = lib.mkForce {
-                garage1 = { zone = "dc1"; capacity = 2000; };
-                garage2 = { zone = "dc1"; capacity = 1000; };
-                garage3 = { zone = "dc1"; capacity = 1000; };
-              };
-              keys.testKey.allow.createBucket = true;
-              buckets = {
-                bucket1 = {
-                  allow.testKey = [ "read" "write" ];
-                  quotas = {
-                    maxObjects = 300;
-                    maxSize = 400 * 1024 * 1024;
-                  };
-                };
-                bucket2 = {
-                  allow.testKey = [ "read" ];
-                };
-              };
-            };
-            system.ascensions.garage-layout.incantations = lib.mkForce (i: [
-              (i.runGarage ''
-                garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
-                garage layout apply --version 2
-              '')
-            ]);
-          };
-        };
-      };
-    };
-  in {
-    garage1.imports = [ common ];
-    garage2.imports = [ common ];
-    garage3.imports = [ common ];
-  };
-
-  testScript = { nodes, ... }: /*python*/ ''
-    nodes = [garage1, garage2, garage3]
-
-    start_all()
-
-    with subtest("should bootstrap new cluster"):
-      for node in nodes:
-        node.wait_for_unit("garage.service")
-
-      for node in nodes:
-        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
-
-    with subtest("should apply new layout with ascension"):
-      for node in nodes:
-        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
-
-      for node in nodes:
-        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
-
-      for node in nodes:
-        node.wait_until_succeeds("garage layout show | grep -w 2000")
-        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
-        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
-
-    with subtest("should apply new layout from scratch"):
-      for node in nodes:
-        node.systemctl("stop garage.service")
-        node.succeed("rm -rf /var/lib/garage-metadata")
-
-      for node in nodes:
-        node.systemctl("start garage.service")
-
-      for node in nodes:
-        node.wait_for_unit("garage.service")
-
-      for node in nodes:
-        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
-
-      for node in nodes:
-        node.wait_until_succeeds("garage layout show | grep -w 2000")
-        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
-        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
-
-    with subtest("should create specified buckets and keys"):
-      for node in nodes:
-        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
-      garage1.succeed("garage key list | grep testKey")
-      garage1.succeed("garage bucket list | grep bucket1")
-      garage1.succeed("garage bucket list | grep bucket2")
-
-    with subtest("should delete unspecified buckets and keys"):
-      garage1.succeed("garage bucket create unwantedbucket")
-      garage1.succeed("garage key new --name unwantedkey")
-      garage1.succeed("systemctl restart garage-apply.service")
-
-      garage1.fail("garage key list | grep unwantedkey")
-      garage1.fail("garage bucket list | grep unwantedbucket")
-  '';
-}

packages/checks/s3ql-upgrade.nix

@@ -10,6 +10,9 @@ testers.runNixOSTest {
       nixosModules.systemd-extras
       ./modules/nixos/age-dummy-secrets
       ./modules/nixos/age-dummy-secrets/options.nix
+      {
+        options.services.locksmith = lib.mkSinkUndeclaredOptions { };
+      }
     ];
 
     _module.args.depot.packages = { inherit (previous.packages.${system}) s3ql; };

packages/patched-inputs.nix

@@ -11,8 +11,6 @@
       nix-super = packages.nix-super.nix;
 
       agenix = packages.agenix.agenix.override { nix = nix-super; };
-
-      hci = packages.hercules-ci-agent.hercules-ci-cli;
     };
   };
 }

packages/projects.nix

@@ -59,7 +59,6 @@
         tools = with flakePkgs; [
           agenix
           graf
-          hci
           npins
           pin
           void

packages/system-filter.nix

@@ -1,6 +1,5 @@
 {
   packages = {
-    hci = [ "x86_64-linux" ];
     hydra = [ "x86_64-linux" ];
     jellyfin = [ "x86_64-linux" ];
     keycloak = [ "x86_64-linux" ];