cluster/services/attic: enable @resources syscall group

`flake.lock`: Update

cluster/lib/mesh.nix

@@ -3,12 +3,14 @@
 {
   hostLinks = lib.pipe config.services [
     (lib.filterAttrs (_: svc: svc.meshLinks != {}))
-    (lib.mapAttrsToList (svcName: svc: lib.mapAttrsToList (name: cfg: lib.genAttrs svc.nodes.${name} (hostName: {
+    (lib.mapAttrsToList (svcName: svc:
-      ${cfg.name} = { ... }: {
+      lib.mapAttrsToList (groupName: links:
+        lib.genAttrs svc.nodes.${groupName} (hostName: lib.mapAttrs (_: cfg: { ... }: {
           imports = [ cfg.link ];
           ipv4 = config.vars.mesh.${hostName}.meshIp;
-      };
+        }) links)
-    })) svc.meshLinks))
+      ) svc.meshLinks
+    ))
     (map lib.mkMerge)
     lib.mkMerge
   ];

cluster/lib/service-module.nix

@@ -38,12 +38,8 @@ in
     };
     meshLinks = mkOption {
       description = "Create host links on the mesh network.";
-      type = types.attrsOf (types.submodule ({ name, ... }: {
+      type = types.attrsOf (types.attrsOf (types.submodule {
         options = {
-          name = mkOption {
-            type = types.str;
-            default = "${serviceName}-${name}";
-          };
           link = mkOption {
             type = types.deferredModule;
             default = {};

cluster/secrets/forge-dbCredentials.age

@@ -1,13 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
-JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
--> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
-2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
--> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
-bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
--> Ovax-grease ^1$]}H G4 FpDF XKHkj{
-IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
---- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
-ÜA³×oÈð:±­‹`ÜVd±å(Kät:fk¼’}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69i›l`ÛÆJKwAè8­y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
-üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ý’wÊ
-ÆÜßÌZî!^þRˆéÿv­¾…ïk‹Êp»ÛPÌ)ý̆ÍpÓV5²F΄ÆÚÙÚÞhBÇ»ßb#Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè‹}€ß	ƒ¬E}~ZHûjmyq{òxŠ–Éôß<C3B4>"”éÀ´C#šójÿÐ.ò§yÔ£¸v¦
<0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që

cluster/services/acme-client/client.nix

@@ -74,7 +74,7 @@ in
       serviceConfig = {
         Restart = "on-failure";
         RestartMaxDelaySec = 30;
-        RestartStesp = 5;
+        RestartSteps = 5;
         RestartMode = "direct";
       };
     };

cluster/services/attic/default.nix

@@ -16,10 +16,7 @@
         ./nar-serve.nix
       ];
     };
-    meshLinks.server = {
+    meshLinks.server.attic.link.protocol = "http";
-      name = "attic";
-      link.protocol = "http";
-    };
     secrets = let
       inherit (config.services.attic) nodes;
     in {

cluster/services/attic/server.nix

@@ -9,17 +9,13 @@ let
 in
 
 {
-  imports = [
-    depot.inputs.attic.nixosModules.atticd
-  ];
-
   services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];
 
   services.atticd = {
     enable = true;
     package = depot.inputs.attic.packages.attic-server;
 
-    credentialsFile = secrets.serverToken.path;
+    environmentFile = secrets.serverToken.path;
     mode = if isMonolith then "monolithic" else "api-server";
 
     settings = {
@@ -69,6 +65,7 @@ in
     serviceConfig = {
       DynamicUser = lib.mkForce false;
       RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
+      SystemCallFilter = lib.mkAfter [ "@resources" ];
     };
     environment = {
       AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
@@ -80,6 +77,7 @@ in
     mode = if isMonolith then "manual" else "direct";
     definition = {
       name = "atticd";
+      id = "atticd-${config.services.atticd.mode}";
       address = link.ipv4;
       inherit (link) port;
       checks = [

cluster/services/forge/default.nix

@@ -4,16 +4,12 @@
   services.forge = {
     nodes.server = [ "VEGAS" ];
     nixos.server = ./server.nix;
-    meshLinks.server = {
+    meshLinks.server.forge.link.protocol = "http";
-      name = "forge";
-      link.protocol = "http";
-    };
     secrets = with config.services.forge.nodes; {
       oidcSecret = {
         nodes = server;
         owner = "forgejo";
       };
-      dbCredentials.nodes = server;
     };
   };
 
@@ -23,6 +19,14 @@
     forge.target = config.hostLinks.${host}.forge.url;
   };
 
+  patroni = config.lib.forService "forge" {
+    databases.forge = {};
+    users.forge.locksmith = {
+      nodes = config.services.forge.nodes.server;
+      format = "raw";
+    };
+  };
+
   garage = config.lib.forService "forge" {
     keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
     buckets.forgejo.allow.forgejo = [ "read" "write" ];

cluster/services/forge/server.nix

@@ -26,6 +26,7 @@ in
   services.locksmith.waitForSecrets.forgejo = [
     "garage-forgejo-id"
     "garage-forgejo-secret"
+    "patroni-forge"
   ];
 
   services.forgejo = {
@@ -39,7 +40,7 @@ in
       inherit (patroni) port;
       name = "forge";
       user = "forge";
-      passwordFile = secrets.dbCredentials.path;
+      passwordFile = "/run/locksmith/patroni-forge";
     };
     settings = {
       DEFAULT = {

cluster/services/idm/server.nix

@@ -33,6 +33,9 @@ in
       ldapbindaddress = "${ldapLink.ipv4}:${ldapLink.portStr}";
       origin = frontendLink.url;
       inherit domain;
+      online_backup = {
+        versions = 7;
+      };
     };
   };
 

cluster/services/ipfs/default.nix

@@ -29,10 +29,7 @@
       io-tweaks = [ "VEGAS" ];
       remote-api = [ "VEGAS" ];
     };
-    meshLinks.gateway = {
+    meshLinks.gateway.ipfsGateway.link.protocol = "http";
-      name = "ipfsGateway";
-      link.protocol = "http";
-    };
     nixos = {
       node = [
         ./node.nix

cluster/services/ipfs/monitoring.nix

@@ -9,7 +9,7 @@ in
     environment = {
       OTEL_TRACES_EXPORTER = "otlp";
       OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-      OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
+      OTEL_EXPORTER_OTLP_ENDPOINT = "${cluster.config.ways.ingest-traces-otlp.url}:443";
       OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
       OTEL_TRACES_SAMPLER_ARG = "0.50";
     };

cluster/services/monitoring/client.nix

@@ -26,7 +26,7 @@ in {
         name = "logging";
         positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
         clients = singleton {
-          url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
+          url = "${cluster.config.ways.ingest-logs.url}/loki/api/v1/push";
         };
         scrape_configs = singleton {
           job_name = "journal";

cluster/services/monitoring/default.nix

@@ -18,26 +18,6 @@ in
       protocol = "http";
       ipv4 = meshIpFor "server";
     };
-    tempo = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
-    tempo-grpc = {
-      protocol = "http";
-      ipv4 = "127.0.0.1";
-    };
-    tempo-otlp-http = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
-    tempo-otlp-grpc = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
-    tempo-zipkin-http = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
   };
   hostLinks = lib.genAttrs config.services.monitoring.nodes.grafana (name: {
     grafana = {
@@ -51,6 +31,7 @@ in
       blackbox = [ "checkmate" "grail" "prophet" ];
       grafana = [ "VEGAS" "prophet" ];
       logging = [ "VEGAS" "grail" ];
+      tracing = [ "VEGAS" "grail" ];
       server = [ "VEGAS" ];
     };
     nixos = {
@@ -61,14 +42,19 @@ in
         ./provisioning/dashboards.nix
       ];
       logging = ./logging.nix;
+      tracing = ./tracing.nix;
       server = [
         ./server.nix
-        ./tracing.nix
       ];
     };
-    meshLinks.logging = {
+    meshLinks = {
-      name = "loki";
+      logging.loki.link.protocol = "http";
-      link.protocol = "http";
+      tracing = {
+        tempo.link.protocol = "http";
+        tempo-otlp-http.link.protocol = "http";
+        tempo-otlp-grpc.link.protocol = "grpc";
+        tempo-zipkin-http.link.protocol = "http";
+      };
     };
   };
 
@@ -82,29 +68,51 @@ in
         nodes = config.services.monitoring.nodes.logging;
         format = "envFile";
       };
-      tempo = { };
+      tempo-ingest.locksmith = {
+        nodes = config.services.monitoring.nodes.tracing;
+        format = "envFile";
+      };
+      tempo-query.locksmith = {
+        nodes = config.services.monitoring.nodes.tracing;
+        format = "envFile";
+      };
     };
     buckets = {
       loki-chunks.allow = {
         loki-ingest = [ "read" "write" ];
         loki-query = [ "read" ];
       };
-      tempo-chunks.allow.tempo = [ "read" "write" ];
+      tempo-chunks.allow = {
+        tempo-ingest = [ "read" "write" ];
+        tempo-query = [ "read" ];
+      };
     };
   };
 
-  ways = config.lib.forService "monitoring" {
+  ways = let
-    monitoring = {
+    query = consulService: {
-      consulService = "grafana";
+      inherit consulService;
-      extras.locations."/".proxyWebsockets = true;
+      internal = true;
-    };
+      extras.extraConfig = ''
-    monitoring-logs = {
+        proxy_read_timeout 3600s;
+      '';
+    };
+    ingest = consulService: {
+      inherit consulService;
       internal = true;
-      consulService = "loki";
       extras.extraConfig = ''
         client_max_body_size 4G;
         proxy_read_timeout 3600s;
       '';
     };
+  in config.lib.forService "monitoring" {
+    monitoring = {
+      consulService = "grafana";
+      extras.locations."/".proxyWebsockets = true;
+    };
+    monitoring-logs = query "loki";
+    monitoring-traces = query "tempo";
+    ingest-logs = ingest "loki";
+    ingest-traces-otlp = ingest "tempo-ingest-otlp-grpc" // { grpc = true; };
   };
 }

cluster/services/monitoring/grafana-ha.nix

@@ -73,6 +73,16 @@ in
           inherit (cluster.config.ways.monitoring-logs) url;
           type = "loki";
         }
+        {
+          name = "Tempo";
+          uid = "P214B5B846CF3925F";
+          inherit (cluster.config.ways.monitoring-traces) url;
+          type = "tempo";
+          jsonData = {
+            serviceMap.datasourceUid = "PBFA97CFB590B2093";
+            nodeGraph.enabled = true;
+          };
+        }
       ];
     };
   };

cluster/services/monitoring/provisioning/objects/dashboards/dashboard-dc7545ef-3180-4a5e-a289-1e64571ebb87.json

@@ -19,10 +19,23 @@
     "editable": true,
     "fiscalYearStartMonth": 0,
     "graphTooltip": 0,
-    "id": 16,
+    "id": 1210,
     "links": [],
     "liveNow": false,
     "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 7,
+        "panels": [],
+        "title": "Replication",
+        "type": "row"
+      },
       {
         "datasource": {
           "type": "prometheus",
@@ -58,7 +71,7 @@
           "h": 8,
           "w": 12,
           "x": 0,
-          "y": 0
+          "y": 1
         },
         "id": 2,
         "options": {
@@ -66,6 +79,7 @@
           "graphMode": "area",
           "justifyMode": "auto",
           "orientation": "auto",
+          "percentChangeColorMode": "standard",
           "reduceOptions": {
             "calcs": [
               "lastNotNull"
@@ -73,9 +87,11 @@
             "fields": "",
             "values": false
           },
-          "textMode": "auto"
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
         },
-        "pluginVersion": "9.5.1",
+        "pluginVersion": "11.1.3",
         "targets": [
           {
             "datasource": {
@@ -138,7 +154,7 @@
           "h": 8,
           "w": 4,
           "x": 12,
-          "y": 0
+          "y": 1
         },
         "id": 3,
         "options": {
@@ -146,14 +162,17 @@
           "graphMode": "area",
           "justifyMode": "auto",
           "orientation": "auto",
+          "percentChangeColorMode": "standard",
           "reduceOptions": {
             "calcs": [],
             "fields": "/^instance$/",
             "values": true
           },
-          "textMode": "auto"
+          "showPercentChange": false,
+          "textMode": "auto",
+          "wideLayout": true
         },
-        "pluginVersion": "9.5.1",
+        "pluginVersion": "11.1.3",
         "targets": [
           {
             "datasource": {
@@ -198,7 +217,7 @@
           "h": 8,
           "w": 8,
           "x": 16,
-          "y": 0
+          "y": 1
         },
         "id": 5,
         "options": {
@@ -254,6 +273,7 @@
               "mode": "palette-classic"
             },
             "custom": {
+              "axisBorderShow": false,
               "axisCenteredZero": false,
               "axisColorMode": "text",
               "axisLabel": "",
@@ -267,6 +287,7 @@
                 "tooltip": false,
                 "viz": false
               },
+              "insertNulls": false,
               "lineInterpolation": "smooth",
               "lineStyle": {
                 "fill": "solid"
@@ -304,7 +325,7 @@
           "h": 13,
           "w": 12,
           "x": 0,
-          "y": 8
+          "y": 9
         },
         "id": 1,
         "options": {
@@ -348,6 +369,7 @@
               "mode": "palette-classic"
             },
             "custom": {
+              "axisBorderShow": false,
               "axisCenteredZero": false,
               "axisColorMode": "text",
               "axisLabel": "",
@@ -361,6 +383,7 @@
                 "tooltip": false,
                 "viz": false
               },
+              "insertNulls": false,
               "lineInterpolation": "smooth",
               "lineWidth": 1,
               "pointSize": 5,
@@ -398,7 +421,7 @@
           "h": 13,
           "w": 12,
           "x": 12,
-          "y": 8
+          "y": 9
         },
         "id": 4,
         "options": {
@@ -428,24 +451,326 @@
         ],
         "title": "Activity",
         "type": "timeseries"
+      },
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 22
+        },
+        "id": 8,
+        "panels": [],
+        "title": "Storage",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              }
+            },
+            "mappings": [],
+            "unit": "bytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 15,
+          "w": 8,
+          "x": 0,
+          "y": 23
+        },
+        "id": 6,
+        "options": {
+          "displayLabels": [
+            "percent"
+          ],
+          "legend": {
+            "calcs": [],
+            "displayMode": "table",
+            "placement": "right",
+            "showLegend": true,
+            "sortBy": "Value",
+            "sortDesc": true,
+            "values": [
+              "percent",
+              "value"
+            ]
+          },
+          "pieType": "donut",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "pluginVersion": "11.1.3",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "disableTextWrap": false,
+            "editorMode": "builder",
+            "exemplar": false,
+            "expr": "max by(datname) (sum by(datname, instance) (pg_database_size_bytes))",
+            "fullMetaSearch": false,
+            "includeNullMetadata": false,
+            "instant": true,
+            "legendFormat": "__auto",
+            "range": false,
+            "refId": "A",
+            "useBackend": false
+          }
+        ],
+        "title": "Database Size",
+        "type": "piechart"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 25,
+              "gradientMode": "opacity",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "bytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 15,
+          "w": 5,
+          "x": 8,
+          "y": 23
+        },
+        "id": 9,
+        "options": {
+          "legend": {
+            "calcs": [
+              "lastNotNull",
+              "max"
+            ],
+            "displayMode": "table",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "pluginVersion": "11.1.3",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "disableTextWrap": false,
+            "editorMode": "builder",
+            "exemplar": false,
+            "expr": "pg_wal_size_bytes",
+            "format": "time_series",
+            "fullMetaSearch": false,
+            "includeNullMetadata": true,
+            "instant": false,
+            "legendFormat": "{{instance}}",
+            "range": true,
+            "refId": "A",
+            "useBackend": false
+          }
+        ],
+        "title": "WAL Size",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds",
+              "seriesBy": "last"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 25,
+              "gradientMode": "opacity",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineStyle": {
+                "fill": "solid"
+              },
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "decimals": 3,
+            "fieldMinMax": false,
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                }
+              ]
+            },
+            "unit": "bytes"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 15,
+          "w": 11,
+          "x": 13,
+          "y": 23
+        },
+        "id": 10,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "disableTextWrap": false,
+            "editorMode": "builder",
+            "exemplar": false,
+            "expr": "max(sum by(instance) (pg_database_size_bytes))",
+            "fullMetaSearch": false,
+            "includeNullMetadata": false,
+            "instant": false,
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A",
+            "useBackend": false
+          }
+        ],
+        "title": "Total DB Size",
+        "type": "timeseries"
       }
     ],
     "refresh": "5m",
-    "schemaVersion": 38,
+    "schemaVersion": 39,
-    "style": "dark",
     "tags": [],
     "templating": {
       "list": []
     },
     "time": {
-      "from": "now-1h",
+      "from": "now-24h",
       "to": "now"
     },
     "timepicker": {},
     "timezone": "",
     "title": "PostgreSQL HA",
     "uid": "dc7545ef-3180-4a5e-a289-1e64571ebb87",
-    "version": 7,
+    "version": 2,
     "weekStart": ""
   },
   "folderId": 0,

cluster/services/monitoring/secrets/tempo-secrets.age

@@ -1,16 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A KhCGp7PAMGrEdzRxBrsW4tRk30JwpI+4lPzrRCUhSw4
-8s7WqA5c3zS1euN5R+jfFNBdvr8OQW8P4NFeqtNsIKo
--> ssh-ed25519 5/zT0w 79hJQ2H76EZTW7YcQFCtKaS5Kbssx4Z8dPFjIVzRgFk
-A1fDJbUnyIRy+kWa3PhJNj/SdRPlcEy6FYsAfnuZ2AQ
--> ssh-ed25519 d3WGuA aylkdL1KliM1NfrYDGlG8X6YjXvVUCU4sV90I+a840U
-6sXdqIPjtoNSylZRh1DCghHOwDo+fC7WB4QWQoWmG48
--> //gd+2-grease baUWA$3 z-qs3W O/2.1W
-Sfq3+rkMJhpUTTmcos5TaaUtX2Ip9pciHAZLiWPix+C9N7ccac/1W5RNedMJCLsq
-MQ+xKzexf8+hgNVhKOksvbKBBROXqk1bUOKk8w3OgFPmmByzmCBUwkdkeu5DFTYR
-rg
---- kUl1uIPRkM5y7C68kdN22pMKXP7gazyha4PE+ap0Jqw
-w>×Àè¥
-<15>CÈ,\‰ßœI¯ˆúHxG@^Çá“PåÃþÙÏlw6
µŽ{þ’rb‘é5æ†T>Êñ
-ÚWܤX4Kp(ß?9ˆß­^^oP3f </v3N$ê¤sÓbŽ¾>O™÷œ+òN0άïµàDtêŽ5Vº#è ¶³‘Uã îŸ#y|›@ŒGzSi»­ô*·HùüŽ]
-ꎀ5›

cluster/services/monitoring/tracing.nix

@@ -1,14 +1,16 @@
 { cluster, config, pkgs, ... }:
 
 let
-  inherit (cluster.config) links;
+  inherit (cluster.config.links) prometheus-ingest;
+  inherit (config.links) tempo-grpc;
+  links = cluster.config.hostLinks.${config.networking.hostName};
   dataDir = "/srv/storage/private/tempo";
   tempoConfig = {
     server = {
       http_listen_address = links.tempo.ipv4;
       http_listen_port = links.tempo.port;
-      grpc_listen_address = links.tempo-grpc.ipv4;
+      grpc_listen_address = tempo-grpc.ipv4;
-      grpc_listen_port = links.tempo-grpc.port;
+      grpc_listen_port = tempo-grpc.port;
     };
     distributor.receivers = {
       otlp = {
@@ -19,7 +21,7 @@ let
       };
       zipkin.endpoint = links.tempo-zipkin-http.tuple;
     };
-    querier.frontend_worker.frontend_address = links.tempo-grpc.tuple;
+    querier.frontend_worker.frontend_address = tempo-grpc.tuple;
     ingester = {
       trace_idle_period = "30s";
       max_block_bytes = 1000000;
@@ -56,7 +58,7 @@ let
         path = "${dataDir}/generator/wal";
         remote_write = [
           {
-            url = "${links.prometheus-ingest.url}/api/v1/write";
+            url = "${prometheus-ingest.url}/api/v1/write";
             send_exemplars = true;
           }
         ];
@@ -68,7 +70,11 @@ let
     ];
   };
 in {
-  age.secrets.tempoSecrets.file = ./secrets/tempo-secrets.age;
+  links.tempo-grpc.protocol = "http";
+
+  services.locksmith.waitForSecrets.tempo = [
+    "garage-tempo-ingest"
+  ];
 
   users.users.tempo = {
     isSystemUser = true;
@@ -81,24 +87,53 @@ in {
 
   systemd.services.tempo = {
     wantedBy = [ "multi-user.target" ];
+    distributed = {
+      enable = true;
+      registerServices = [
+        "tempo"
+        "tempo-ingest-otlp-grpc"
+      ];
+    };
     serviceConfig = {
       User = "tempo";
       Group = "tempo";
       ExecStart = "${pkgs.tempo}/bin/tempo -config.file=${pkgs.writeText "tempo.yaml" (builtins.toJSON tempoConfig)}";
       PrivateTmp = true;
-      EnvironmentFile = config.age.secrets.tempoSecrets.path;
+      EnvironmentFile = "/run/locksmith/garage-tempo-ingest";
     };
   };
-  services.grafana.provision.datasources.settings.datasources = [
+
+  consul.services = {
+    tempo = {
+      mode = "manual";
+      definition = {
+        name = "tempo";
+        address = links.tempo.ipv4;
+        inherit (links.tempo) port;
+        checks = [
           {
             name = "Tempo";
-      uid = "P214B5B846CF3925F";
+            id = "service:tempo:backend";
-      inherit (links.tempo) url;
+            interval = "5s";
-      type = "tempo";
+            http = "${links.tempo.url}/ready";
-      jsonData = {
-        serviceMap.datasourceUid = "PBFA97CFB590B2093"; # prometheus
-        nodeGraph.enabled = true;
-      };
           }
         ];
+      };
+    };
+    tempo-ingest-otlp-grpc = {
+      mode = "manual";
+      definition = {
+        name = "tempo-ingest-otlp-grpc";
+        address = links.tempo-otlp-grpc.ipv4;
+        inherit (links.tempo-otlp-grpc) port;
+        checks = [
+          {
+            name = "Tempo Service Status";
+            id = "service:tempo-ingest-otlp-grpc:tempo";
+            alias_service = "tempo";
+          }
+        ];
+      };
+    };
+  };
 }

cluster/services/sso/default.nix

@@ -1,4 +1,4 @@
-{ depot, ... }:
+{ config, depot, ... }:
 
 {
   services.sso = {
@@ -18,4 +18,12 @@
     login.target = ssoAddr;
     account.target = ssoAddr;
   };
+
+  patroni = config.lib.forService "sso" {
+    databases.keycloak = {};
+    users.keycloak.locksmith = {
+      nodes = config.services.sso.nodes.host;
+      format = "raw";
+    };
+  };
 }

cluster/services/sso/host.nix

@@ -8,12 +8,10 @@ in
 {
   links.keycloak.protocol = "http";
 
-  age.secrets.keycloak-dbpass = {
+  services.locksmith.waitForSecrets.keycloak = [
-    file = ../../../secrets/keycloak-dbpass.age;
+    "patroni-keycloak"
-    owner = "root";
+  ];
-    group = "root";
+
-    mode = "0400";
-  };
   services.nginx.virtualHosts = { 
     "${login}" = lib.recursiveUpdate (vhosts.proxy kc.url) {
       locations = {
@@ -36,13 +34,14 @@ in
       host = patroni.ipv4;
       inherit (patroni) port;
       useSSL = false;
-      passwordFile = config.age.secrets.keycloak-dbpass.path;
+      passwordFile = "/run/locksmith/patroni-keycloak";
     };
     settings = {
+      http-enabled = true;
       http-host = kc.ipv4;
       http-port = kc.port;
       hostname = login;
-      proxy = "edge";
+      proxy-headers = "xforwarded";
       # for backcompat, TODO: remove
       http-relative-path = "/auth";
     };
@@ -54,7 +53,7 @@ in
       "-Dotel.traces.exporter=otlp"
     ];
     OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
+    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.ways.ingest-traces-otlp.url;
     OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
     OTEL_TRACES_SAMPLER_ARG = "0.50";
   };

cluster/services/storage/default.nix

@@ -56,7 +56,7 @@ in
     };
     simulacrum = {
       enable = true;
-      deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
+      deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" "ways" ];
       settings = ./simulacrum/test.nix;
     };
   };

cluster/services/storage/garage-gateway.nix

@@ -36,10 +36,9 @@ in
         inherit (linkWeb) port;
         checks = [
           {
-            name = "Garage Node";
+            name = "Garage Service Status";
-            id = "service:garage-web:node";
+            id = "service:garage-web:garage";
-            interval = "5s";
+            alias_service = "garage";
-            http = "${config.links.garageMetrics.url}/health";
           }
         ];
       };

cluster/services/ways/host.nix

@@ -25,7 +25,14 @@ in
             ];
             locations = lib.mkMerge [
               {
-                "/".proxyPass = cfg.target;
+                "/" = if cfg.grpc then {
+                  extraConfig = ''
+                    set $nix_proxy_grpc_target ${cfg.target};
+                    grpc_pass $nix_proxy_grpc_target;
+                  '';
+                } else {
+                  proxyPass = cfg.target;
+                };
                 "${cfg.healthCheckPath}".extraConfig = "access_log off;";
               }
               {
@@ -82,11 +89,11 @@ in
             '') consulServiceWays;
           in pkgs.writeText "ways-upstreams.ctmpl" (lib.concatStringsSep "\n" (lib.unique upstreams));
           destination = "/run/consul-template/nginx-ways-upstreams.conf";
-          exec.command = [
+          exec.command = lib.singleton (pkgs.writeShellScript "ways-reload" ''
-            "${config.services.nginx.package}/bin/nginx"
+            if ${config.systemd.package}/bin/systemctl is-active nginx.service; then
-            "-s" "reload"
+              exec ${config.services.nginx.package}/bin/nginx -s reload -g 'pid /run/nginx/nginx.pid;'
-            "-g" "pid /run/nginx/nginx.pid;"
+            fi
-          ];
+          '');
         }
       ];
     };

cluster/services/ways/options/way.nix

@@ -35,6 +35,12 @@ with lib;
       };
     };
 
+    grpc = mkOption {
+      description = "Whether this endpoint is a gRPC service.";
+      type = types.bool;
+      default = false;
+    };
+
     target = mkOption {
       type = types.str;
     };
@@ -101,7 +107,7 @@ with lib;
     (lib.mkIf options.consulService.isDefined {
       useConsul = true;
       nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
-      target = "http://${options.nginxUpstreamName.value}";
+      target = "${if config.grpc then "grpc" else "http"}://${options.nginxUpstreamName.value}";
     })
     (lib.mkIf options.bucket.isDefined {
       consulService = "garage-web";

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1722339003,
+        "lastModified": 1723293904,
-        "narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
         "type": "github"
       },
       "original": {
@@ -29,9 +29,8 @@
         "flake-compat": [
           "blank"
         ],
-        "flake-utils": [
+        "flake-parts": "flake-parts",
-          "repin-flake-utils"
+        "nix-github-actions": "nix-github-actions",
-        ],
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -40,11 +39,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1722472866,
+        "lastModified": 1730906442,
-        "narHash": "sha256-GJIz4M5HDB948Ex/8cPvbkrNzl/eKUE7/c21JBu4lb8=",
+        "narHash": "sha256-tBuyb8jWBSHHgcIrOfiyQJZGY1IviMzH2V74t7gWfgI=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "e127acbf9a71ebc0c26bc8e28346822e0a6e16ba",
+        "rev": "d0b66cf897e4d55f03d341562c9821dc4e566e54",
         "type": "github"
       },
       "original": {
@@ -76,11 +75,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717025063,
+        "lastModified": 1722960479,
-        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
         "type": "github"
       },
       "original": {
@@ -118,11 +117,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1722113426,
+        "lastModified": 1728330715,
-        "narHash": "sha256-Yo/3loq572A8Su6aY5GP56knpuKYRvM2a1meP9oJZCw=",
+        "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "67cce7359e4cd3c45296fb4aaf6a19e2a9c757ae",
+        "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
         "type": "github"
       },
       "original": {
@@ -190,6 +189,7 @@
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
+          "attic",
           "nixpkgs"
         ]
       },
@@ -208,6 +208,26 @@
       }
     },
     "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1730504689,
+        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_3": {
       "inputs": {
         "nixpkgs-lib": [
           "nix-super",
@@ -268,11 +288,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1720223941,
+        "lastModified": 1723736589,
-        "narHash": "sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4=",
+        "narHash": "sha256-/Vdg5ZKtP71ZEKVV6JXlrOEu0CM2Flcs+nwDmWRzgjQ=",
         "owner": "hercules-ci",
         "repo": "hercules-ci-agent",
-        "rev": "2e10fb21fc2e07edf40763b73443e5934bd40947",
+        "rev": "c303cc8e437c0fd26b9452472e7df5aa374e9177",
         "type": "github"
       },
       "original": {
@@ -291,11 +311,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1719226092,
+        "lastModified": 1730903510,
-        "narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=",
+        "narHash": "sha256-mnynlrPeiW0nUQ8KGZHb3WyxAxA3Ye/BH8gMjdoKP6E=",
         "owner": "hercules-ci",
         "repo": "hercules-ci-effects",
-        "rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5",
+        "rev": "b89ac4d66d618b915b1f0a408e2775fe3821d141",
         "type": "github"
       },
       "original": {
@@ -399,11 +419,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1722409392,
+        "lastModified": 1725623828,
-        "narHash": "sha256-8QuMS00EutmqzAIPxyJEPxM8EHiWlSKs6E2Htoh3Kes=",
+        "narHash": "sha256-5Zrn72PO9yBaNO4Gd5uOsEmRpYH5rVAFKOQ5h2PxyhU=",
         "owner": "numtide",
         "repo": "nar-serve",
-        "rev": "9d0eff868d328fe67c60c26c8ba50e0b9d8de867",
+        "rev": "e5c749a444f2d14f381c75ef3a8feaa82c333b92",
         "type": "github"
       },
       "original": {
@@ -414,11 +434,11 @@
     },
     "nix-filter": {
       "locked": {
-        "lastModified": 1710156097,
+        "lastModified": 1730207686,
-        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
         "type": "github"
       },
       "original": {
@@ -427,10 +447,31 @@
         "type": "github"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nix-super": {
       "inputs": {
         "flake-compat": "flake-compat_2",
-        "flake-parts": "flake-parts_2",
+        "flake-parts": "flake-parts_3",
         "libgit2": "libgit2",
         "nixpkgs": "nixpkgs_3",
         "nixpkgs-regression": [
@@ -504,11 +545,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1722539632,
+        "lastModified": 1730785428,
-        "narHash": "sha256-g4L+I8rDl7RQy5x8XcEMqNO49LFhrHTzVBqXtG2+FGo=",
+        "narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f2d6c7123138044e0c68902268bd8f37dd7e2fa7",
+        "rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
         "type": "github"
       },
       "original": {
@@ -557,11 +598,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1710146030,
+        "lastModified": 1726560853,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -577,7 +618,7 @@
         "blank": "blank",
         "devshell": "devshell",
         "drv-parts": "drv-parts",
-        "flake-parts": "flake-parts",
+        "flake-parts": "flake-parts_2",
         "hercules-ci-agent": "hercules-ci-agent",
         "hercules-ci-effects": "hercules-ci-effects",
         "hyprspace": "hyprspace",

flake.nix

@@ -54,7 +54,6 @@
         nixpkgs.follows = "nixpkgs";
         nixpkgs-stable.follows = "nixpkgs";
         flake-compat.follows = "blank";
-        flake-utils.follows = "repin-flake-utils";
       };
     };
 

modules/consul-distributed-services/default.nix

@@ -26,11 +26,13 @@ in
 
       cfg = v.distributed;
 
-      svc = config.consul.services.${cfg.registerService};
+      svcs = map (x: config.consul.services.${x}) cfg.registerServices;
 
       runWithRegistration = pkgs.writeShellScript "run-with-registration" ''
-        trap '${svc.commands.deregister}' EXIT
+        trap '${lib.concatStringsSep ";" (map (svc: svc.commands.deregister) svcs)}' EXIT
-        ${svc.commands.register}
+        ${lib.concatStringsSep "\n" (
+          map (svc: svc.commands.register) svcs
+        )}
         ''${@}
       '';
 
@@ -49,10 +51,10 @@ in
       [Service]
       ExecStartPre=${waitForConsul} 'services/${n}%i'
       ExecStart=
-      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
+      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerServices != []) runWithRegistration} ${ExecStart}
       Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
       ${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
-      ${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
+      ${optionalString (cfg.registerServices != []) (lib.concatStringsSep "\n" (map (svc: "ExecStopPost=${svc.commands.deregister}") svcs))}
     ''))
   ];
 }

modules/systemd-extras/distributed.nix

@@ -17,6 +17,11 @@ with lib;
           type = with types; nullOr str;
           default = null;
         };
+        registerServices = mkOption {
+          description = "Consul services to register when this service gets started.";
+          type = with types; listOf str;
+          default = if config.distributed.registerService == null then [ ] else [ config.distributed.registerService ];
+        };
       };
     }));
   };

packages/checks/keycloak-custom-jre.nix

@@ -7,7 +7,8 @@ nixosTest {
     package = keycloak;
     database.passwordFile = builtins.toFile "keycloak-test-password" "kcnixostest1234";
     settings = {
-      proxy = "edge";
+      http-enabled = true;
+      proxy-headers = "xforwarded";
       hostname = "keycloak.local";
     };
   };

packages/patched-derivations.nix

@@ -18,6 +18,10 @@ super: rec {
     };
   };
 
+  jitsi-meet-insecure = let
+    olm-insecure = assert builtins.length super.olm.meta.knownVulnerabilities > 0; super.olm.overrideAttrs (o: { meta = o.meta // { knownVulnerabilities = []; }; });
+  in super.jitsi-meet.override { olm = olm-insecure; };
+
   jre17_standard = let
     jre = super.jre_minimal.override {
       jdk = super.jdk17_headless;
@@ -46,7 +50,7 @@ super: rec {
 
   prometheus-jitsi-exporter = patch super.prometheus-jitsi-exporter "patches/base/prometheus-jitsi-exporter";
 
-  s3ql = (patch super.s3ql "patches/base/s3ql").overrideAttrs (old: {
+  s3ql = super.s3ql.overrideAttrs (old: {
     propagatedBuildInputs = old.propagatedBuildInputs ++ [
       super.python3Packages.packaging
       super.python3Packages.systemd

packages/shadows.nix

@@ -9,7 +9,9 @@
 
     options.shadows = lib.mkOption {
       type = with lib.types; lazyAttrsOf package;
-      default = { };
+      default = {
+        jitsi-meet = self'.packages.jitsi-meet-insecure;
+      };
     };
   };
 }

patches/base/kanidm/unixd-authenticated.patch

@@ -1,25 +1,28 @@
-diff --git a/unix_integration/src/idprovider/kanidm.rs b/unix_integration/src/idprovider/kanidm.rs
+diff --git a/unix_integration/resolver/src/idprovider/kanidm.rs b/unix_integration/resolver/src/idprovider/kanidm.rs
-index 6fc015756..31593f03e 100644
+index 63cedb4d5..35c45fb0e 100644
---- a/unix_integration/src/idprovider/kanidm.rs
+--- a/unix_integration/resolver/src/idprovider/kanidm.rs
-+++ b/unix_integration/src/idprovider/kanidm.rs
++++ b/unix_integration/resolver/src/idprovider/kanidm.rs
-@@ -4,6 +4,7 @@ use kanidm_client::{ClientError, KanidmClient, StatusCode};
+@@ -7,6 +7,7 @@ use kanidm_proto::internal::OperationError;
- use kanidm_proto::internal::OperationError;
  use kanidm_proto::v1::{UnixGroupToken, UnixUserToken};
- use tokio::sync::{broadcast, RwLock};
+ use std::collections::BTreeSet;
+ use std::time::{Duration, SystemTime};
 +use std::env;
+ use tokio::sync::{broadcast, Mutex};
  
- use super::interface::{
+ use kanidm_lib_crypto::CryptoPolicy;
-     // KeyStore,
+@@ -38,6 +39,8 @@ struct KanidmProviderInternal {
-@@ -25,12 +26,28 @@ const TAG_IDKEY: &str = "idkey";
+     hmac_key: HmacKey,
- 
+     crypto_policy: CryptoPolicy,
- pub struct KanidmProvider {
+     pam_allow_groups: BTreeSet<String>,
-     client: RwLock<KanidmClient>,
 +    auth_name: Option<String>,
 +    auth_password: Option<String>,
  }
  
- impl KanidmProvider {
+ pub struct KanidmProvider {
-     pub fn new(client: KanidmClient) -> Self {
+@@ -102,6 +105,19 @@ impl KanidmProvider {
+             .map(|GroupMap { local, with }| (local, Id::Name(with)))
+             .collect();
+ 
 +        let env_username: Option<String>;
 +        let env_password: Option<String>;
 +        match (env::var_os("KANIDM_NAME"), env::var_os("KANIDM_PASSWORD")) {
@@ -32,23 +35,29 @@ index 6fc015756..31593f03e 100644
 +                env_password = None;
 +            }
 +        }
-         KanidmProvider {
++
-             client: RwLock::new(client),
+         Ok(KanidmProvider {
+             inner: Mutex::new(KanidmProviderInternal {
+                 state: CacheState::OfflineNextCheck(now),
+@@ -109,6 +125,8 @@ impl KanidmProvider {
+                 hmac_key,
+                 crypto_policy,
+                 pam_allow_groups,
 +                auth_name: env_username,
-+            auth_password: env_password,
++                auth_password: env_password
+             }),
+             map_group,
+         })
+@@ -256,7 +274,11 @@ impl KanidmProviderInternal {
      }
-     }
- }
-@@ -118,7 +135,11 @@ impl IdProvider for KanidmProvider {
  
-     // Needs .read on all types except re-auth.
+     async fn attempt_online(&mut self, _tpm: &mut tpm::BoxedDynTpm, now: SystemTime) -> bool {
-     async fn provider_authenticate(&self, _tpm: &mut tpm::BoxedDynTpm) -> Result<(), IdpError> {
+-        match self.client.auth_anonymous().await {
--        match self.client.write().await.auth_anonymous().await {
 +        let auth_method = match (&self.auth_name, &self.auth_password) {
-+            (Some(name), Some(password)) => self.client.write().await.auth_simple_password(name, password).await,
++            (Some(name), Some(password)) => self.client.auth_simple_password(name, password).await,
-+            _ => self.client.write().await.auth_anonymous().await
++            _ => self.client.auth_anonymous().await
 +        };
 +        match auth_method {
-             Ok(_uat) => Ok(()),
+             Ok(_uat) => {
-             Err(err) => {
+                 self.state = CacheState::Online;
-                 error!(?err, "Provider authentication failed");
+                 true

patches/base/s3ql/0000-cache-entry-seek-whence.patch

@@ -1,18 +0,0 @@
-diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py
-index a4b55fd1..267b9a12 100644
---- a/src/s3ql/block_cache.py
-+++ b/src/s3ql/block_cache.py
-@@ -86,10 +86,10 @@ class CacheEntry:
-     def flush(self):
-         self.fh.flush()
- 
--    def seek(self, off):
-+    def seek(self, off, whence=0):
-         if self.pos != off:
--            self.fh.seek(off)
--            self.pos = off
-+            self.fh.seek(off, whence)
-+            self.pos = self.fh.tell()
- 
-     def tell(self):
-         return self.pos

patches/base/s3ql/0001-fix-plain-block-size.patch

@@ -1,26 +0,0 @@
-diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
-index 6402fec1..9ed3627e 100644
---- a/src/s3ql/backends/comprenc.py
-+++ b/src/s3ql/backends/comprenc.py
-@@ -276,7 +276,7 @@ class ComprencBackend(AbstractBackend):
-             buf.seek(0)
-             fh = buf
- 
--        return self.backend.write_fh(key, fh, meta_raw)
-+        return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
- 
-     def contains(self, key):
-         return self.backend.contains(key)
-diff --git a/src/s3ql/database.py b/src/s3ql/database.py
-index bb4054e6..c2142bf6 100644
---- a/src/s3ql/database.py
-+++ b/src/s3ql/database.py
-@@ -659,7 +659,7 @@ def upload_metadata(
-             )
-             obj = METADATA_OBJ_NAME % (blockno, params.seq_no)
-             fh.seek(blockno * blocksize)
--            backend.write_fh(obj, fh, len_=blocksize)
-+            backend.write_fh(obj, fh, len_=min(blocksize, db_size - blockno * blocksize))
- 
-         if not update_params:
-             return

patches/base/s3ql/0002-comprenc-always-copy.patch

@@ -1,17 +0,0 @@
-diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
-index 9ed3627e..db419bb7 100644
---- a/src/s3ql/backends/comprenc.py
-+++ b/src/s3ql/backends/comprenc.py
-@@ -276,6 +276,12 @@ class ComprencBackend(AbstractBackend):
-             buf.seek(0)
-             fh = buf
- 
-+        if meta_raw['compression'] == 'None' and meta_raw['encryption'] == 'None':
-+            buf = io.BytesIO()
-+            copyfh(fh, buf, len_)
-+            buf.seek(0)
-+            fh = buf
-+        
-         return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
- 
-     def contains(self, key):

patches/base/s3ql/remove-ssl-monkeypatch.patch

@@ -1,12 +0,0 @@
-diff --git a/tests/t0_http.py b/tests/t0_http.py
-index 66ed564f..36bebab1 100755
---- a/tests/t0_http.py
-+++ b/tests/t0_http.py
-@@ -289,7 +289,6 @@ def do_GET(self):
- 
-     # We don't *actually* want to establish SSL, that'd be
-     # to complex for our mock server
--    monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
-     conn = HTTPConnection(
-         test_host,
-         test_port,

patches/base/s3ql/s3c-accurate-length.patch

@@ -1,26 +0,0 @@
-commit 1edbbcf08d5701ea38f13fca7491418318aebca9
-Author: Max <max@privatevoid.net>
-Date:   Fri Jun 7 23:31:08 2024 +0200
-
-    accurate length
-
-diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py
-index 2995ca4f..3c3c79ab 100644
---- a/src/s3ql/backends/s3c.py
-+++ b/src/s3ql/backends/s3c.py
-@@ -387,9 +387,13 @@ class Backend(AbstractBackend):
-         '''
- 
-         off = fh.tell()
-+        fh.seek(0, os.SEEK_END)
-+        actual_len = fh.tell() - off
-+        fh.seek(off, os.SEEK_SET)
-         if len_ is None:
--            fh.seek(0, os.SEEK_END)
--            len_ = fh.tell()
-+            len_ = actual_len
-+        else:
-+            len_ = min(len_, actual_len)
-         return self._write_fh(key, fh, off, len_, metadata or {})
- 
-     @retry

patches/base/s3ql/s3v4.patch

@@ -1,392 +0,0 @@
-From 11e3a9cea77cd8498d874f7fd69a938af4da68cd Mon Sep 17 00:00:00 2001
-From: xeji <36407913+xeji@users.noreply.github.com>
-Date: Thu, 28 Mar 2024 22:19:11 +0100
-Subject: [PATCH] new backend s3c4: s3c with V4 request signatures (#349)
-
----
- rst/backends.rst              |  15 ++++
- src/s3ql/backends/__init__.py |   3 +-
- src/s3ql/backends/s3.py       | 100 ++----------------------
- src/s3ql/backends/s3c4.py     | 140 ++++++++++++++++++++++++++++++++++
- src/s3ql/parse_args.py        |   2 +-
- tests/mock_server.py          |  11 +++
- 6 files changed, 174 insertions(+), 97 deletions(-)
- create mode 100644 src/s3ql/backends/s3c4.py
-
-diff --git a/rst/backends.rst b/rst/backends.rst
-index 7220ee96..4bc68387 100644
---- a/rst/backends.rst
-+++ b/rst/backends.rst
-@@ -341,6 +341,14 @@ can be an arbitrary prefix that will be prepended to all object names
- used by S3QL. This allows you to store several S3QL file systems in
- the same bucket.
- 
-+`s3c://` authenticates API requests using AWS V2 signatures, which are
-+deprecated by AWS but still accepted by many S3 compatible services.
-+
-+`s3c4://` denotes a variant of this backend that works the same
-+but uses AWS V4 signatures for request authentication instead: ::
-+
-+   s3c4://<hostname>:<port>/<bucketname>/<prefix>
-+
- The S3 compatible backend accepts the following backend options:
- 
- .. option:: no-ssl
-@@ -385,6 +393,13 @@ The S3 compatible backend accepts the following backend options:
-    necessary if your storage server does not return a valid response
-    body for a successful copy operation.
- 
-+.. option:: sig-region=<region>
-+
-+   For `s3c4://` variant only: Region to use for calculating V4
-+   request signatures. Contrary to S3, the region is not a defined 
-+   part of the storage URL and must be specified separately.
-+   Defaults to `us-east-1`.
-+
- .. _`S3 COPY API`: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
- .. __: https://doc.s3.amazonaws.com/proposals/copy.html
- 
-diff --git a/src/s3ql/backends/__init__.py b/src/s3ql/backends/__init__.py
-index a1335762..442828cd 100644
---- a/src/s3ql/backends/__init__.py
-+++ b/src/s3ql/backends/__init__.py
-@@ -6,7 +6,7 @@
- This work can be distributed under the terms of the GNU GPLv3.
- '''
- 
--from . import gs, local, rackspace, s3, s3c, swift, swiftks
-+from . import gs, local, rackspace, s3, s3c, s3c4, swift, swiftks
- from .b2.b2_backend import B2Backend
- 
- #: Mapping from storage URL prefixes to backend classes
-@@ -15,6 +15,7 @@
-     'local': local.Backend,
-     'gs': gs.Backend,
-     's3c': s3c.Backend,
-+    's3c4': s3c4.Backend,
-     'swift': swift.Backend,
-     'swiftks': swiftks.Backend,
-     'rackspace': rackspace.Backend,
-diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py
-index e05a49ba..5548a855 100644
---- a/src/s3ql/backends/s3.py
-+++ b/src/s3ql/backends/s3.py
-@@ -15,7 +15,7 @@
- from xml.sax.saxutils import escape as xml_escape
- 
- from ..logging import QuietError
--from . import s3c
-+from . import s3c4
- from .common import retry
- from .s3c import get_S3Error
- 
-@@ -28,22 +28,23 @@
- # pylint: disable=E1002,E1101
- 
- 
--class Backend(s3c.Backend):
-+class Backend(s3c4.Backend):
-     """A backend to store data in Amazon S3
- 
-     This class uses standard HTTP connections to connect to S3.
-     """
- 
--    known_options = (s3c.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
-+    known_options = (s3c4.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
-         'dumb-copy',
-         'disable-expect100',
-+        'sig-region',
-     }
- 
-     def __init__(self, options):
-         self.region = None
--        self.signing_key = None
-         super().__init__(options)
-         self._set_storage_options(self._extra_put_headers)
-+        self.sig_region = self.region
- 
-     def _parse_storage_url(self, storage_url, ssl_context):
-         hit = re.match(r'^s3s?://([^/]+)/([^/]+)(?:/(.*))?$', storage_url)
-@@ -147,94 +148,3 @@ def _delete_multi(self, keys):
- 
-         except:
-             self.conn.discard()
--
--    def _authorize_request(self, method, path, headers, subres, query_string):
--        '''Add authorization information to *headers*'''
--
--        # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
--
--        now = time.gmtime()
--        # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
--        #                    '%a, %d %b %Y %H:%M:%S GMT')
--
--        ymd = time.strftime('%Y%m%d', now)
--        ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
--
--        headers['x-amz-date'] = ymdhms
--        headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
--        # headers['x-amz-content-sha256'] = hashlib.sha256(body).hexdigest()
--        headers.pop('Authorization', None)
--
--        auth_strs = [method]
--        auth_strs.append(urllib.parse.quote(path))
--
--        if query_string:
--            s = urllib.parse.urlencode(
--                query_string, doseq=True, quote_via=urllib.parse.quote
--            ).split('&')
--        else:
--            s = []
--        if subres:
--            s.append(urllib.parse.quote(subres) + '=')
--        if s:
--            s = '&'.join(sorted(s))
--        else:
--            s = ''
--        auth_strs.append(s)
--
--        # Headers
--        sig_hdrs = sorted(x.lower() for x in headers.keys())
--        for hdr in sig_hdrs:
--            auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
--        auth_strs.append('')
--        auth_strs.append(';'.join(sig_hdrs))
--        auth_strs.append(headers['x-amz-content-sha256'])
--        can_req = '\n'.join(auth_strs)
--        # log.debug('canonical request: %s', can_req)
--
--        can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
--        str_to_sign = (
--            "AWS4-HMAC-SHA256\n"
--            + ymdhms
--            + '\n'
--            + '%s/%s/s3/aws4_request\n' % (ymd, self.region)
--            + can_req_hash
--        )
--        # log.debug('string to sign: %s', str_to_sign)
--
--        if self.signing_key is None or self.signing_key[1] != ymd:
--            self.update_signing_key(ymd)
--        signing_key = self.signing_key[0]
--
--        sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
--
--        cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
--            self.login,
--            now.tm_year,
--            now.tm_mon,
--            now.tm_mday,
--            self.region,
--        )
--
--        headers['Authorization'] = (
--            'AWS4-HMAC-SHA256 '
--            'Credential=%s,'
--            'SignedHeaders=%s,'
--            'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
--        )
--
--    def update_signing_key(self, ymd):
--        date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
--        region_key = hmac_sha256(date_key, self.region.encode())
--        service_key = hmac_sha256(region_key, b's3')
--        signing_key = hmac_sha256(service_key, b'aws4_request')
--
--        self.signing_key = (signing_key, ymd)
--
--
--def hmac_sha256(key, msg, hex=False):
--    d = hmac.new(key, msg, hashlib.sha256)
--    if hex:
--        return d.hexdigest()
--    else:
--        return d.digest()
-diff --git a/src/s3ql/backends/s3c4.py b/src/s3ql/backends/s3c4.py
-new file mode 100644
-index 00000000..37ff0b7a
---- /dev/null
-+++ b/src/s3ql/backends/s3c4.py
-@@ -0,0 +1,140 @@
-+'''
-+s3c4.py - this file is part of S3QL.
-+
-+Copyright © 2008 Nikolaus Rath <Nikolaus@rath.org>
-+
-+This work can be distributed under the terms of the GNU GPLv3.
-+'''
-+
-+import hashlib
-+import hmac
-+import logging
-+import re
-+import time
-+import urllib.parse
-+from xml.sax.saxutils import escape as xml_escape
-+
-+from ..logging import QuietError
-+from . import s3c
-+from .common import retry
-+from .s3c import get_S3Error
-+
-+log = logging.getLogger(__name__)
-+
-+# Maximum number of keys that can be deleted at once
-+MAX_KEYS = 1000
-+
-+# Pylint goes berserk with false positives
-+# pylint: disable=E1002,E1101
-+
-+
-+class Backend(s3c.Backend):
-+    """A backend to stored data in some S3 compatible storage service.
-+
-+    This classes uses AWS Signature V4 for authorization.
-+    """
-+
-+    known_options = s3c.Backend.known_options | {'sig-region'}
-+
-+    def __init__(self, options):
-+        self.sig_region = options.backend_options.get('sig-region', 'us-east-1')
-+        self.signing_key = None
-+        super().__init__(options)
-+
-+    def __str__(self):
-+        return 's3c4://%s/%s/%s' % (self.hostname, self.bucket_name, self.prefix)
-+
-+    def _authorize_request(self, method, path, headers, subres, query_string):
-+        '''Add authorization information to *headers*'''
-+
-+        # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
-+
-+        now = time.gmtime()
-+        # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
-+        #                    '%a, %d %b %Y %H:%M:%S GMT')
-+
-+        ymd = time.strftime('%Y%m%d', now)
-+        ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
-+
-+        # add non-standard port to host header, needed for correct signature
-+        if self.port != 443:
-+            headers['host'] = '%s:%s' % (self.hostname, self.port)
-+
-+        headers['x-amz-date'] = ymdhms
-+        headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
-+
-+        headers.pop('Authorization', None)
-+
-+        auth_strs = [method]
-+        auth_strs.append(urllib.parse.quote(path))
-+
-+        if query_string:
-+            s = urllib.parse.urlencode(
-+                query_string, doseq=True, quote_via=urllib.parse.quote
-+            ).split('&')
-+        else:
-+            s = []
-+        if subres:
-+            s.append(urllib.parse.quote(subres) + '=')
-+        if s:
-+            s = '&'.join(sorted(s))
-+        else:
-+            s = ''
-+        auth_strs.append(s)
-+
-+        # Headers
-+        sig_hdrs = sorted(x.lower() for x in headers.keys())
-+        for hdr in sig_hdrs:
-+            auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
-+        auth_strs.append('')
-+        auth_strs.append(';'.join(sig_hdrs))
-+        auth_strs.append(headers['x-amz-content-sha256'])
-+        can_req = '\n'.join(auth_strs)
-+        # log.debug('canonical request: %s', can_req)
-+
-+        can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
-+        str_to_sign = (
-+            "AWS4-HMAC-SHA256\n"
-+            + ymdhms
-+            + '\n'
-+            + '%s/%s/s3/aws4_request\n' % (ymd, self.sig_region)
-+            + can_req_hash
-+        )
-+        # log.debug('string to sign: %s', str_to_sign)
-+
-+        if self.signing_key is None or self.signing_key[1] != ymd:
-+            self.update_signing_key(ymd)
-+        signing_key = self.signing_key[0]
-+
-+        sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
-+
-+        cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
-+            self.login,
-+            now.tm_year,
-+            now.tm_mon,
-+            now.tm_mday,
-+            self.sig_region,
-+        )
-+
-+        headers['Authorization'] = (
-+            'AWS4-HMAC-SHA256 '
-+            'Credential=%s,'
-+            'SignedHeaders=%s,'
-+            'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
-+        )
-+
-+    def update_signing_key(self, ymd):
-+        date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
-+        region_key = hmac_sha256(date_key, self.sig_region.encode())
-+        service_key = hmac_sha256(region_key, b's3')
-+        signing_key = hmac_sha256(service_key, b'aws4_request')
-+
-+        self.signing_key = (signing_key, ymd)
-+
-+
-+def hmac_sha256(key, msg, hex=False):
-+    d = hmac.new(key, msg, hashlib.sha256)
-+    if hex:
-+        return d.hexdigest()
-+    else:
-+        return d.digest()
-diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py
-index 272e10c7..24ad50f4 100644
---- a/src/s3ql/parse_args.py
-+++ b/src/s3ql/parse_args.py
-@@ -374,7 +374,7 @@ def storage_url_type(s):
-     # slash (even when using a prefix), but we can't do that now because it
-     # would make file systems created without trailing slash inaccessible.
-     if re.match(r'^(s3|gs)://[^/]+$', s) or re.match(
--        r'^(s3c|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
-+        r'^(s3c|s3c4|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
-     ):
-         s += '/'
- 
-diff --git a/tests/mock_server.py b/tests/mock_server.py
-index b453e705..e3084065 100644
---- a/tests/mock_server.py
-+++ b/tests/mock_server.py
-@@ -292,6 +292,16 @@ def send_error(self, status, message=None, code='', resource='', extra_headers=N
-             self.wfile.write(content)
- 
- 
-+class S3C4RequestHandler(S3CRequestHandler):
-+    '''Request Handler for s3c4 backend
-+
-+    Currently identical to S3CRequestHandler since mock request handlers
-+    do not check request signatures.
-+    '''
-+
-+    pass
-+
-+
- class BasicSwiftRequestHandler(S3CRequestHandler):
-     '''A request handler implementing a subset of the OpenStack Swift Interface
- 
-@@ -569,6 +579,7 @@ def inline_error(http_status, body):
- #: corresponding storage urls
- handler_list = [
-     (S3CRequestHandler, 's3c://%(host)s:%(port)d/s3ql_test'),
-+    (S3C4RequestHandler, 's3c4://%(host)s:%(port)d/s3ql_test'),
-     # Special syntax only for testing against mock server
-     (BasicSwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),
-     (CopySwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),