flake.lock: Update

Flake lock file updates:

• Updated input 'hercules-ci-agent':
    'github:hercules-ci/hercules-ci-agent/2e10fb21fc2e07edf40763b73443e5934bd40947?narHash=sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4%3D' (2024-07-05)
  → 'github:nrabulinski/hercules-ci-agent/c41b58bf33c9167670685eb97ede9ac240b321d4?narHash=sha256-Dx4kNqDE3//22xyICJ6czV465MUvspowZJlLGSbSGpo%3D' (2024-08-15)
• Removed input 'hercules-ci-agent/haskell-flake'

cluster/lib/mesh.nix

@@ -3,14 +3,12 @@
 {
   hostLinks = lib.pipe config.services [
     (lib.filterAttrs (_: svc: svc.meshLinks != {}))
-    (lib.mapAttrsToList (svcName: svc:
-      lib.mapAttrsToList (groupName: links:
-        lib.genAttrs svc.nodes.${groupName} (hostName: lib.mapAttrs (_: cfg: { ... }: {
-          imports = [ cfg.link ];
-          ipv4 = config.vars.mesh.${hostName}.meshIp;
-        }) links)
-      ) svc.meshLinks
-    ))
+    (lib.mapAttrsToList (svcName: svc: lib.mapAttrsToList (name: cfg: lib.genAttrs svc.nodes.${name} (hostName: {
+      ${cfg.name} = { ... }: {
+        imports = [ cfg.link ];
+        ipv4 = config.vars.mesh.${hostName}.meshIp;
+      };
+    })) svc.meshLinks))
     (map lib.mkMerge)
     lib.mkMerge
   ];

cluster/lib/service-module.nix

@@ -38,8 +38,12 @@ in
     };
     meshLinks = mkOption {
       description = "Create host links on the mesh network.";
-      type = types.attrsOf (types.attrsOf (types.submodule {
+      type = types.attrsOf (types.submodule ({ name, ... }: {
         options = {
+          name = mkOption {
+            type = types.str;
+            default = "${serviceName}-${name}";
+          };
           link = mkOption {
             type = types.deferredModule;
             default = {};

cluster/secrets/forge-dbCredentials.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
+JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
+-> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
+2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
+-> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
+bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
+-> Ovax-grease ^1$]}H G4 FpDF XKHkj{
+IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
+--- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
+ÜA³×oÈð:±­‹`ÜVd±å(Kät:fk¼’}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69i›l`ÛÆJKwAè8­y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
+üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ý’wÊ
+ÆÜßÌZî!^þRˆéÿv­¾…ïk‹Êp»ÛPÌ)ý̆ÍpÓV5²F΄ÆÚÙÚÞhBÇ»ßb#Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè‹}€ß	ƒ¬E}~ZHûjmyq{òxŠ–Éôß<C3B4>"”éÀ´C#šójÿÐ.ò§yÔ£¸v¦
<0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që

cluster/services/acme-client/client.nix

@@ -74,7 +74,7 @@ in
       serviceConfig = {
         Restart = "on-failure";
         RestartMaxDelaySec = 30;
-        RestartSteps = 5;
+        RestartStesp = 5;
         RestartMode = "direct";
       };
     };

cluster/services/attic/default.nix

@@ -16,7 +16,10 @@
         ./nar-serve.nix
       ];
     };
-    meshLinks.server.attic.link.protocol = "http";
+    meshLinks.server = {
+      name = "attic";
+      link.protocol = "http";
+    };
     secrets = let
       inherit (config.services.attic) nodes;
     in {

cluster/services/attic/server.nix

@@ -80,7 +80,6 @@ in
     mode = if isMonolith then "manual" else "direct";
     definition = {
       name = "atticd";
-      id = "atticd-${config.services.atticd.mode}";
       address = link.ipv4;
       inherit (link) port;
       checks = [

cluster/services/forge/default.nix

@@ -4,12 +4,16 @@
   services.forge = {
     nodes.server = [ "VEGAS" ];
     nixos.server = ./server.nix;
-    meshLinks.server.forge.link.protocol = "http";
+    meshLinks.server = {
+      name = "forge";
+      link.protocol = "http";
+    };
     secrets = with config.services.forge.nodes; {
       oidcSecret = {
         nodes = server;
         owner = "forgejo";
       };
+      dbCredentials.nodes = server;
     };
   };
 
@@ -19,14 +23,6 @@
     forge.target = config.hostLinks.${host}.forge.url;
   };
 
-  patroni = config.lib.forService "forge" {
-    databases.forge = {};
-    users.forge.locksmith = {
-      nodes = config.services.forge.nodes.server;
-      format = "raw";
-    };
-  };
-
   garage = config.lib.forService "forge" {
     keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
     buckets.forgejo.allow.forgejo = [ "read" "write" ];

cluster/services/forge/server.nix

@@ -26,7 +26,6 @@ in
   services.locksmith.waitForSecrets.forgejo = [
     "garage-forgejo-id"
     "garage-forgejo-secret"
-    "patroni-forge"
   ];
 
   services.forgejo = {
@@ -40,7 +39,7 @@ in
       inherit (patroni) port;
       name = "forge";
       user = "forge";
-      passwordFile = "/run/locksmith/patroni-forge";
+      passwordFile = secrets.dbCredentials.path;
     };
     settings = {
       DEFAULT = {

cluster/services/idm/server.nix

@@ -33,9 +33,6 @@ in
       ldapbindaddress = "${ldapLink.ipv4}:${ldapLink.portStr}";
       origin = frontendLink.url;
       inherit domain;
-      online_backup = {
-        versions = 7;
-      };
     };
   };
 

cluster/services/ipfs/default.nix

@@ -29,7 +29,10 @@
       io-tweaks = [ "VEGAS" ];
       remote-api = [ "VEGAS" ];
     };
-    meshLinks.gateway.ipfsGateway.link.protocol = "http";
+    meshLinks.gateway = {
+      name = "ipfsGateway";
+      link.protocol = "http";
+    };
     nixos = {
       node = [
         ./node.nix

cluster/services/ipfs/monitoring.nix

@@ -9,7 +9,7 @@ in
     environment = {
       OTEL_TRACES_EXPORTER = "otlp";
       OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-      OTEL_EXPORTER_OTLP_ENDPOINT = "${cluster.config.ways.ingest-traces-otlp.url}:443";
+      OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
       OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
       OTEL_TRACES_SAMPLER_ARG = "0.50";
     };

cluster/services/monitoring/client.nix

@@ -26,7 +26,7 @@ in {
         name = "logging";
         positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
         clients = singleton {
-          url = "${cluster.config.ways.ingest-logs.url}/loki/api/v1/push";
+          url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
         };
         scrape_configs = singleton {
           job_name = "journal";

cluster/services/monitoring/default.nix

@@ -18,6 +18,26 @@ in
       protocol = "http";
       ipv4 = meshIpFor "server";
     };
+    tempo = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
+    tempo-grpc = {
+      protocol = "http";
+      ipv4 = "127.0.0.1";
+    };
+    tempo-otlp-http = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
+    tempo-otlp-grpc = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
+    tempo-zipkin-http = {
+      protocol = "http";
+      ipv4 = meshIpFor "server";
+    };
   };
   hostLinks = lib.genAttrs config.services.monitoring.nodes.grafana (name: {
     grafana = {
@@ -31,7 +51,6 @@ in
       blackbox = [ "checkmate" "grail" "prophet" ];
       grafana = [ "VEGAS" "prophet" ];
       logging = [ "VEGAS" "grail" ];
-      tracing = [ "VEGAS" "grail" ];
       server = [ "VEGAS" ];
     };
     nixos = {
@@ -42,19 +61,14 @@ in
         ./provisioning/dashboards.nix
       ];
       logging = ./logging.nix;
-      tracing = ./tracing.nix;
       server = [
         ./server.nix
+        ./tracing.nix
       ];
     };
-    meshLinks = {
-      logging.loki.link.protocol = "http";
-      tracing = {
-        tempo.link.protocol = "http";
-        tempo-otlp-http.link.protocol = "http";
-        tempo-otlp-grpc.link.protocol = "grpc";
-        tempo-zipkin-http.link.protocol = "http";
-      };
+    meshLinks.logging = {
+      name = "loki";
+      link.protocol = "http";
     };
   };
 
@@ -68,51 +82,29 @@ in
         nodes = config.services.monitoring.nodes.logging;
         format = "envFile";
       };
-      tempo-ingest.locksmith = {
-        nodes = config.services.monitoring.nodes.tracing;
-        format = "envFile";
-      };
-      tempo-query.locksmith = {
-        nodes = config.services.monitoring.nodes.tracing;
-        format = "envFile";
-      };
+      tempo = { };
     };
     buckets = {
       loki-chunks.allow = {
         loki-ingest = [ "read" "write" ];
         loki-query = [ "read" ];
       };
-      tempo-chunks.allow = {
-        tempo-ingest = [ "read" "write" ];
-        tempo-query = [ "read" ];
-      };
+      tempo-chunks.allow.tempo = [ "read" "write" ];
     };
   };
 
-  ways = let
-    query = consulService: {
-      inherit consulService;
-      internal = true;
-      extras.extraConfig = ''
-        proxy_read_timeout 3600s;
-      '';
+  ways = config.lib.forService "monitoring" {
+    monitoring = {
+      consulService = "grafana";
+      extras.locations."/".proxyWebsockets = true;
     };
-    ingest = consulService: {
-      inherit consulService;
+    monitoring-logs = {
       internal = true;
+      consulService = "loki";
       extras.extraConfig = ''
         client_max_body_size 4G;
         proxy_read_timeout 3600s;
       '';
     };
-  in config.lib.forService "monitoring" {
-    monitoring = {
-      consulService = "grafana";
-      extras.locations."/".proxyWebsockets = true;
-    };
-    monitoring-logs = query "loki";
-    monitoring-traces = query "tempo";
-    ingest-logs = ingest "loki";
-    ingest-traces-otlp = ingest "tempo-ingest-otlp-grpc" // { grpc = true; };
   };
 }

cluster/services/monitoring/grafana-ha.nix

@@ -73,16 +73,6 @@ in
           inherit (cluster.config.ways.monitoring-logs) url;
           type = "loki";
         }
-        {
-          name = "Tempo";
-          uid = "P214B5B846CF3925F";
-          inherit (cluster.config.ways.monitoring-traces) url;
-          type = "tempo";
-          jsonData = {
-            serviceMap.datasourceUid = "PBFA97CFB590B2093";
-            nodeGraph.enabled = true;
-          };
-        }
       ];
     };
   };

cluster/services/monitoring/provisioning/objects/dashboards/dashboard-dc7545ef-3180-4a5e-a289-1e64571ebb87.json

@@ -19,23 +19,10 @@
     "editable": true,
     "fiscalYearStartMonth": 0,
     "graphTooltip": 0,
-    "id": 1210,
+    "id": 16,
     "links": [],
     "liveNow": false,
     "panels": [
-      {
-        "collapsed": false,
-        "gridPos": {
-          "h": 1,
-          "w": 24,
-          "x": 0,
-          "y": 0
-        },
-        "id": 7,
-        "panels": [],
-        "title": "Replication",
-        "type": "row"
-      },
       {
         "datasource": {
           "type": "prometheus",
@@ -71,7 +58,7 @@
           "h": 8,
           "w": 12,
           "x": 0,
-          "y": 1
+          "y": 0
         },
         "id": 2,
         "options": {
@@ -79,7 +66,6 @@
           "graphMode": "area",
           "justifyMode": "auto",
           "orientation": "auto",
-          "percentChangeColorMode": "standard",
           "reduceOptions": {
             "calcs": [
               "lastNotNull"
@@ -87,11 +73,9 @@
             "fields": "",
             "values": false
           },
-          "showPercentChange": false,
-          "textMode": "auto",
-          "wideLayout": true
+          "textMode": "auto"
         },
-        "pluginVersion": "11.1.3",
+        "pluginVersion": "9.5.1",
         "targets": [
           {
             "datasource": {
@@ -154,7 +138,7 @@
           "h": 8,
           "w": 4,
           "x": 12,
-          "y": 1
+          "y": 0
         },
         "id": 3,
         "options": {
@@ -162,17 +146,14 @@
           "graphMode": "area",
           "justifyMode": "auto",
           "orientation": "auto",
-          "percentChangeColorMode": "standard",
           "reduceOptions": {
             "calcs": [],
             "fields": "/^instance$/",
             "values": true
           },
-          "showPercentChange": false,
-          "textMode": "auto",
-          "wideLayout": true
+          "textMode": "auto"
         },
-        "pluginVersion": "11.1.3",
+        "pluginVersion": "9.5.1",
         "targets": [
           {
             "datasource": {
@@ -217,7 +198,7 @@
           "h": 8,
           "w": 8,
           "x": 16,
-          "y": 1
+          "y": 0
         },
         "id": 5,
         "options": {
@@ -273,7 +254,6 @@
               "mode": "palette-classic"
             },
             "custom": {
-              "axisBorderShow": false,
               "axisCenteredZero": false,
               "axisColorMode": "text",
               "axisLabel": "",
@@ -287,7 +267,6 @@
                 "tooltip": false,
                 "viz": false
               },
-              "insertNulls": false,
               "lineInterpolation": "smooth",
               "lineStyle": {
                 "fill": "solid"
@@ -325,7 +304,7 @@
           "h": 13,
           "w": 12,
           "x": 0,
-          "y": 9
+          "y": 8
         },
         "id": 1,
         "options": {
@@ -369,7 +348,6 @@
               "mode": "palette-classic"
             },
             "custom": {
-              "axisBorderShow": false,
               "axisCenteredZero": false,
               "axisColorMode": "text",
               "axisLabel": "",
@@ -383,7 +361,6 @@
                 "tooltip": false,
                 "viz": false
               },
-              "insertNulls": false,
               "lineInterpolation": "smooth",
               "lineWidth": 1,
               "pointSize": 5,
@@ -421,7 +398,7 @@
           "h": 13,
           "w": 12,
           "x": 12,
-          "y": 9
+          "y": 8
         },
         "id": 4,
         "options": {
@@ -451,326 +428,24 @@
         ],
         "title": "Activity",
         "type": "timeseries"
-      },
-      {
-        "collapsed": false,
-        "gridPos": {
-          "h": 1,
-          "w": 24,
-          "x": 0,
-          "y": 22
-        },
-        "id": 8,
-        "panels": [],
-        "title": "Storage",
-        "type": "row"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              }
-            },
-            "mappings": [],
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 8,
-          "x": 0,
-          "y": 23
-        },
-        "id": 6,
-        "options": {
-          "displayLabels": [
-            "percent"
-          ],
-          "legend": {
-            "calcs": [],
-            "displayMode": "table",
-            "placement": "right",
-            "showLegend": true,
-            "sortBy": "Value",
-            "sortDesc": true,
-            "values": [
-              "percent",
-              "value"
-            ]
-          },
-          "pieType": "donut",
-          "reduceOptions": {
-            "calcs": [
-              "lastNotNull"
-            ],
-            "fields": "",
-            "values": false
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "pluginVersion": "11.1.3",
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "max by(datname) (sum by(datname, instance) (pg_database_size_bytes))",
-            "fullMetaSearch": false,
-            "includeNullMetadata": false,
-            "instant": true,
-            "legendFormat": "__auto",
-            "range": false,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "Database Size",
-        "type": "piechart"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisBorderShow": false,
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 25,
-              "gradientMode": "opacity",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "insertNulls": false,
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            },
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 5,
-          "x": 8,
-          "y": 23
-        },
-        "id": 9,
-        "options": {
-          "legend": {
-            "calcs": [
-              "lastNotNull",
-              "max"
-            ],
-            "displayMode": "table",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "pluginVersion": "11.1.3",
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "pg_wal_size_bytes",
-            "format": "time_series",
-            "fullMetaSearch": false,
-            "includeNullMetadata": true,
-            "instant": false,
-            "legendFormat": "{{instance}}",
-            "range": true,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "WAL Size",
-        "type": "timeseries"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "PBFA97CFB590B2093"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "thresholds",
-              "seriesBy": "last"
-            },
-            "custom": {
-              "axisBorderShow": false,
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 25,
-              "gradientMode": "opacity",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "insertNulls": false,
-              "lineInterpolation": "linear",
-              "lineStyle": {
-                "fill": "solid"
-              },
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "decimals": 3,
-            "fieldMinMax": false,
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                }
-              ]
-            },
-            "unit": "bytes"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 15,
-          "w": 11,
-          "x": 13,
-          "y": 23
-        },
-        "id": 10,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": false
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "PBFA97CFB590B2093"
-            },
-            "disableTextWrap": false,
-            "editorMode": "builder",
-            "exemplar": false,
-            "expr": "max(sum by(instance) (pg_database_size_bytes))",
-            "fullMetaSearch": false,
-            "includeNullMetadata": false,
-            "instant": false,
-            "legendFormat": "__auto",
-            "range": true,
-            "refId": "A",
-            "useBackend": false
-          }
-        ],
-        "title": "Total DB Size",
-        "type": "timeseries"
       }
     ],
     "refresh": "5m",
-    "schemaVersion": 39,
+    "schemaVersion": 38,
+    "style": "dark",
     "tags": [],
     "templating": {
       "list": []
     },
     "time": {
-      "from": "now-24h",
+      "from": "now-1h",
       "to": "now"
     },
     "timepicker": {},
     "timezone": "",
     "title": "PostgreSQL HA",
     "uid": "dc7545ef-3180-4a5e-a289-1e64571ebb87",
-    "version": 2,
+    "version": 7,
     "weekStart": ""
   },
   "folderId": 0,

cluster/services/monitoring/secrets/tempo-secrets.age

@@ -0,0 +1,16 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A KhCGp7PAMGrEdzRxBrsW4tRk30JwpI+4lPzrRCUhSw4
+8s7WqA5c3zS1euN5R+jfFNBdvr8OQW8P4NFeqtNsIKo
+-> ssh-ed25519 5/zT0w 79hJQ2H76EZTW7YcQFCtKaS5Kbssx4Z8dPFjIVzRgFk
+A1fDJbUnyIRy+kWa3PhJNj/SdRPlcEy6FYsAfnuZ2AQ
+-> ssh-ed25519 d3WGuA aylkdL1KliM1NfrYDGlG8X6YjXvVUCU4sV90I+a840U
+6sXdqIPjtoNSylZRh1DCghHOwDo+fC7WB4QWQoWmG48
+-> //gd+2-grease baUWA$3 z-qs3W O/2.1W
+Sfq3+rkMJhpUTTmcos5TaaUtX2Ip9pciHAZLiWPix+C9N7ccac/1W5RNedMJCLsq
+MQ+xKzexf8+hgNVhKOksvbKBBROXqk1bUOKk8w3OgFPmmByzmCBUwkdkeu5DFTYR
+rg
+--- kUl1uIPRkM5y7C68kdN22pMKXP7gazyha4PE+ap0Jqw
+w>×Àè¥
+<15>CÈ,\‰ßœI¯ˆúHxG@^Çá“PåÃþÙÏlw6
µŽ{þ’rb‘é5æ†T>Êñ
+ÚWܤX4Kp(ß?9ˆß­^^oP3f </v3N$ê¤sÓbŽ¾>O™÷œ+òN0άïµàDtêŽ5Vº#è ¶³‘Uã îŸ#y|›@ŒGzSi»­ô*·HùüŽ]
+ꎀ5›

cluster/services/monitoring/tracing.nix

@@ -1,16 +1,14 @@
 { cluster, config, pkgs, ... }:
 
 let
-  inherit (cluster.config.links) prometheus-ingest;
-  inherit (config.links) tempo-grpc;
-  links = cluster.config.hostLinks.${config.networking.hostName};
+  inherit (cluster.config) links;
   dataDir = "/srv/storage/private/tempo";
   tempoConfig = {
     server = {
       http_listen_address = links.tempo.ipv4;
       http_listen_port = links.tempo.port;
-      grpc_listen_address = tempo-grpc.ipv4;
-      grpc_listen_port = tempo-grpc.port;
+      grpc_listen_address = links.tempo-grpc.ipv4;
+      grpc_listen_port = links.tempo-grpc.port;
     };
     distributor.receivers = {
       otlp = {
@@ -21,7 +19,7 @@ let
       };
       zipkin.endpoint = links.tempo-zipkin-http.tuple;
     };
-    querier.frontend_worker.frontend_address = tempo-grpc.tuple;
+    querier.frontend_worker.frontend_address = links.tempo-grpc.tuple;
     ingester = {
       trace_idle_period = "30s";
       max_block_bytes = 1000000;
@@ -58,7 +56,7 @@ let
         path = "${dataDir}/generator/wal";
         remote_write = [
           {
-            url = "${prometheus-ingest.url}/api/v1/write";
+            url = "${links.prometheus-ingest.url}/api/v1/write";
             send_exemplars = true;
           }
         ];
@@ -70,11 +68,7 @@ let
     ];
   };
 in {
-  links.tempo-grpc.protocol = "http";
-
-  services.locksmith.waitForSecrets.tempo = [
-    "garage-tempo-ingest"
-  ];
+  age.secrets.tempoSecrets.file = ./secrets/tempo-secrets.age;
 
   users.users.tempo = {
     isSystemUser = true;
@@ -87,53 +81,24 @@ in {
 
   systemd.services.tempo = {
     wantedBy = [ "multi-user.target" ];
-    distributed = {
-      enable = true;
-      registerServices = [
-        "tempo"
-        "tempo-ingest-otlp-grpc"
-      ];
-    };
     serviceConfig = {
       User = "tempo";
       Group = "tempo";
       ExecStart = "${pkgs.tempo}/bin/tempo -config.file=${pkgs.writeText "tempo.yaml" (builtins.toJSON tempoConfig)}";
       PrivateTmp = true;
-      EnvironmentFile = "/run/locksmith/garage-tempo-ingest";
+      EnvironmentFile = config.age.secrets.tempoSecrets.path;
     };
   };
-
-  consul.services = {
-    tempo = {
-      mode = "manual";
-      definition = {
-        name = "tempo";
-        address = links.tempo.ipv4;
-        inherit (links.tempo) port;
-        checks = [
-          {
-            name = "Tempo";
-            id = "service:tempo:backend";
-            interval = "5s";
-            http = "${links.tempo.url}/ready";
-          }
-        ];
+  services.grafana.provision.datasources.settings.datasources = [
+    {
+      name = "Tempo";
+      uid = "P214B5B846CF3925F";
+      inherit (links.tempo) url;
+      type = "tempo";
+      jsonData = {
+        serviceMap.datasourceUid = "PBFA97CFB590B2093"; # prometheus
+        nodeGraph.enabled = true;
       };
-    };
-    tempo-ingest-otlp-grpc = {
-      mode = "manual";
-      definition = {
-        name = "tempo-ingest-otlp-grpc";
-        address = links.tempo-otlp-grpc.ipv4;
-        inherit (links.tempo-otlp-grpc) port;
-        checks = [
-          {
-            name = "Tempo Service Status";
-            id = "service:tempo-ingest-otlp-grpc:tempo";
-            alias_service = "tempo";
-          }
-        ];
-      };
-    };
-  };
+    }
+  ];
 }

cluster/services/sso/default.nix

@@ -1,4 +1,4 @@
-{ config, depot, ... }:
+{ depot, ... }:
 
 {
   services.sso = {
@@ -18,12 +18,4 @@
     login.target = ssoAddr;
     account.target = ssoAddr;
   };
-
-  patroni = config.lib.forService "sso" {
-    databases.keycloak = {};
-    users.keycloak.locksmith = {
-      nodes = config.services.sso.nodes.host;
-      format = "raw";
-    };
-  };
 }

cluster/services/sso/host.nix

@@ -8,10 +8,12 @@ in
 {
   links.keycloak.protocol = "http";
 
-  services.locksmith.waitForSecrets.keycloak = [
-    "patroni-keycloak"
-  ];
-
+  age.secrets.keycloak-dbpass = {
+    file = ../../../secrets/keycloak-dbpass.age;
+    owner = "root";
+    group = "root";
+    mode = "0400";
+  };
   services.nginx.virtualHosts = { 
     "${login}" = lib.recursiveUpdate (vhosts.proxy kc.url) {
       locations = {
@@ -34,7 +36,7 @@ in
       host = patroni.ipv4;
       inherit (patroni) port;
       useSSL = false;
-      passwordFile = "/run/locksmith/patroni-keycloak";
+      passwordFile = config.age.secrets.keycloak-dbpass.path;
     };
     settings = {
       http-host = kc.ipv4;
@@ -52,7 +54,7 @@ in
       "-Dotel.traces.exporter=otlp"
     ];
     OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.ways.ingest-traces-otlp.url;
+    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
     OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
     OTEL_TRACES_SAMPLER_ARG = "0.50";
   };

cluster/services/storage/garage-gateway.nix

@@ -36,9 +36,10 @@ in
         inherit (linkWeb) port;
         checks = [
           {
-            name = "Garage Service Status";
-            id = "service:garage-web:garage";
-            alias_service = "garage";
+            name = "Garage Node";
+            id = "service:garage-web:node";
+            interval = "5s";
+            http = "${config.links.garageMetrics.url}/health";
           }
         ];
       };

cluster/services/ways/host.nix

@@ -25,14 +25,7 @@ in
             ];
             locations = lib.mkMerge [
               {
-                "/" = if cfg.grpc then {
-                  extraConfig = ''
-                    set $nix_proxy_grpc_target ${cfg.target};
-                    grpc_pass $nix_proxy_grpc_target;
-                  '';
-                } else {
-                  proxyPass = cfg.target;
-                };
+                "/".proxyPass = cfg.target;
                 "${cfg.healthCheckPath}".extraConfig = "access_log off;";
               }
               {

cluster/services/ways/options/way.nix

@@ -35,12 +35,6 @@ with lib;
       };
     };
 
-    grpc = mkOption {
-      description = "Whether this endpoint is a gRPC service.";
-      type = types.bool;
-      default = false;
-    };
-
     target = mkOption {
       type = types.str;
     };
@@ -107,7 +101,7 @@ with lib;
     (lib.mkIf options.consulService.isDefined {
       useConsul = true;
       nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
-      target = "${if config.grpc then "grpc" else "http"}://${options.nginxUpstreamName.value}";
+      target = "http://${options.nginxUpstreamName.value}";
     })
     (lib.mkIf options.bucket.isDefined {
       consulService = "garage-web";

flake.lock

@@ -243,40 +243,24 @@
         "type": "github"
       }
     },
-    "haskell-flake": {
-      "locked": {
-        "lastModified": 1684780604,
-        "narHash": "sha256-2uMZsewmRn7rRtAnnQNw1lj0uZBMh4m6Cs/7dV5YF08=",
-        "owner": "srid",
-        "repo": "haskell-flake",
-        "rev": "74210fa80a49f1b6f67223debdbf1494596ff9f2",
-        "type": "github"
-      },
-      "original": {
-        "owner": "srid",
-        "ref": "0.3.0",
-        "repo": "haskell-flake",
-        "type": "github"
-      }
-    },
     "hercules-ci-agent": {
       "inputs": {
         "flake-parts": [
           "flake-parts"
         ],
-        "haskell-flake": "haskell-flake",
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1720223941,
-        "narHash": "sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4=",
-        "owner": "hercules-ci",
+        "lastModified": 1723760507,
+        "narHash": "sha256-Dx4kNqDE3//22xyICJ6czV465MUvspowZJlLGSbSGpo=",
+        "owner": "nrabulinski",
         "repo": "hercules-ci-agent",
-        "rev": "2e10fb21fc2e07edf40763b73443e5934bd40947",
+        "rev": "c41b58bf33c9167670685eb97ede9ac240b321d4",
         "type": "github"
       },
       "original": {
-        "owner": "hercules-ci",
+        "owner": "nrabulinski",
+        "ref": "ifdless",
         "repo": "hercules-ci-agent",
         "type": "github"
       }

flake.nix

@@ -83,7 +83,7 @@
     };
     
     hercules-ci-agent = {
-      url = "github:hercules-ci/hercules-ci-agent";
+      url = "github:nrabulinski/hercules-ci-agent/ifdless";
       inputs = {
         flake-parts.follows = "flake-parts";
       };

modules/consul-distributed-services/default.nix

@@ -26,13 +26,11 @@ in
 
       cfg = v.distributed;
 
-      svcs = map (x: config.consul.services.${x}) cfg.registerServices;
+      svc = config.consul.services.${cfg.registerService};
 
       runWithRegistration = pkgs.writeShellScript "run-with-registration" ''
-        trap '${lib.concatStringsSep ";" (map (svc: svc.commands.deregister) svcs)}' EXIT
-        ${lib.concatStringsSep "\n" (
-          map (svc: svc.commands.register) svcs
-        )}
+        trap '${svc.commands.deregister}' EXIT
+        ${svc.commands.register}
         ''${@}
       '';
 
@@ -51,10 +49,10 @@ in
       [Service]
       ExecStartPre=${waitForConsul} 'services/${n}%i'
       ExecStart=
-      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerServices != []) runWithRegistration} ${ExecStart}
+      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
       Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
       ${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
-      ${optionalString (cfg.registerServices != []) (lib.concatStringsSep "\n" (map (svc: "ExecStopPost=${svc.commands.deregister}") svcs))}
+      ${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
     ''))
   ];
 }

modules/systemd-extras/distributed.nix

@@ -17,11 +17,6 @@ with lib;
           type = with types; nullOr str;
           default = null;
         };
-        registerServices = mkOption {
-          description = "Consul services to register when this service gets started.";
-          type = with types; listOf str;
-          default = if config.distributed.registerService == null then [ ] else [ config.distributed.registerService ];
-        };
       };
     }));
   };