cluster/services/sso: use correct tempo address

cluster/services/ipfs/monitoring.nix

@@ -9,7 +9,7 @@ in
     environment = {
       OTEL_TRACES_EXPORTER = "otlp";
       OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-      OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
+      OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.ways.ingest-traces-otlp.url;
       OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
       OTEL_TRACES_SAMPLER_ARG = "0.50";
     };

cluster/services/monitoring/default.nix

@@ -18,26 +18,6 @@ in
       protocol = "http";
       ipv4 = meshIpFor "server";
     };
-    tempo = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
-    tempo-grpc = {
-      protocol = "http";
-      ipv4 = "127.0.0.1";
-    };
-    tempo-otlp-http = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
-    tempo-otlp-grpc = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
-    tempo-zipkin-http = {
-      protocol = "http";
-      ipv4 = meshIpFor "server";
-    };
   };
   hostLinks = lib.genAttrs config.services.monitoring.nodes.grafana (name: {
     grafana = {
@@ -51,6 +31,7 @@ in
       blackbox = [ "checkmate" "grail" "prophet" ];
       grafana = [ "VEGAS" "prophet" ];
       logging = [ "VEGAS" "grail" ];
+      tracing = [ "VEGAS" "grail" ];
       server = [ "VEGAS" ];
     };
     nixos = {
@@ -61,12 +42,20 @@ in
         ./provisioning/dashboards.nix
       ];
       logging = ./logging.nix;
+      tracing = ./tracing.nix;
       server = [
         ./server.nix
-        ./tracing.nix
       ];
     };
-    meshLinks.logging.loki.link.protocol = "http";
+    meshLinks = {
+      logging.loki.link.protocol = "http";
+      tracing = {
+        tempo.link.protocol = "http";
+        tempo-otlp-http.link.protocol = "http";
+        tempo-otlp-grpc.link.protocol = "grpc";
+        tempo-zipkin-http.link.protocol = "http";
+      };
+    };
   };
 
   garage = config.lib.forService "monitoring" {
@@ -79,36 +68,51 @@ in
         nodes = config.services.monitoring.nodes.logging;
         format = "envFile";
       };
-      tempo = { };
+      tempo-ingest.locksmith = {
+        nodes = config.services.monitoring.nodes.tracing;
+        format = "envFile";
+      };
+      tempo-query.locksmith = {
+        nodes = config.services.monitoring.nodes.tracing;
+        format = "envFile";
+      };
     };
     buckets = {
       loki-chunks.allow = {
         loki-ingest = [ "read" "write" ];
         loki-query = [ "read" ];
       };
-      tempo-chunks.allow.tempo = [ "read" "write" ];
+      tempo-chunks.allow = {
+        tempo-ingest = [ "read" "write" ];
+        tempo-query = [ "read" ];
+      };
     };
   };
 
-  ways = config.lib.forService "monitoring" {
-    monitoring = {
-      consulService = "grafana";
-      extras.locations."/".proxyWebsockets = true;
-    };
-    monitoring-logs = {
+  ways = let
+    query = consulService: {
+      inherit consulService;
       internal = true;
-      consulService = "loki";
       extras.extraConfig = ''
         proxy_read_timeout 3600s;
       '';
     };
-    ingest-logs = {
+    ingest = consulService: {
+      inherit consulService;
       internal = true;
-      consulService = "loki";
       extras.extraConfig = ''
         client_max_body_size 4G;
         proxy_read_timeout 3600s;
       '';
     };
+  in config.lib.forService "monitoring" {
+    monitoring = {
+      consulService = "grafana";
+      extras.locations."/".proxyWebsockets = true;
+    };
+    monitoring-logs = query "loki";
+    monitoring-traces = query "tempo";
+    ingest-logs = ingest "loki";
+    ingest-traces-otlp = ingest "tempo-ingest-otlp-grpc" // { grpc = true; };
   };
 }

cluster/services/monitoring/grafana-ha.nix

@@ -73,6 +73,16 @@ in
           inherit (cluster.config.ways.monitoring-logs) url;
           type = "loki";
         }
+        {
+          name = "Tempo";
+          uid = "P214B5B846CF3925F";
+          inherit (cluster.config.ways.monitoring-traces) url;
+          type = "tempo";
+          jsonData = {
+            serviceMap.datasourceUid = "PBFA97CFB590B2093";
+            nodeGraph.enabled = true;
+          };
+        }
       ];
     };
   };

cluster/services/monitoring/secrets/tempo-secrets.age

@@ -1,16 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 NO562A KhCGp7PAMGrEdzRxBrsW4tRk30JwpI+4lPzrRCUhSw4
-8s7WqA5c3zS1euN5R+jfFNBdvr8OQW8P4NFeqtNsIKo
--> ssh-ed25519 5/zT0w 79hJQ2H76EZTW7YcQFCtKaS5Kbssx4Z8dPFjIVzRgFk
-A1fDJbUnyIRy+kWa3PhJNj/SdRPlcEy6FYsAfnuZ2AQ
--> ssh-ed25519 d3WGuA aylkdL1KliM1NfrYDGlG8X6YjXvVUCU4sV90I+a840U
-6sXdqIPjtoNSylZRh1DCghHOwDo+fC7WB4QWQoWmG48
--> //gd+2-grease baUWA$3 z-qs3W O/2.1W
-Sfq3+rkMJhpUTTmcos5TaaUtX2Ip9pciHAZLiWPix+C9N7ccac/1W5RNedMJCLsq
-MQ+xKzexf8+hgNVhKOksvbKBBROXqk1bUOKk8w3OgFPmmByzmCBUwkdkeu5DFTYR
-rg
---- kUl1uIPRkM5y7C68kdN22pMKXP7gazyha4PE+ap0Jqw
-w>×Àè¥
-<15>CÈ,\‰ßœI¯ˆúHxG@^Çá“PåÃþÙÏlw6
µŽ{þ’rb‘é5æ†T>Êñ
-ÚWܤX4Kp(ß?9ˆß­^^oP3f </v3N$ê¤sÓbŽ¾>O™÷œ+òN0άïµàDtêŽ5Vº#è ¶³‘Uã îŸ#y|›@ŒGzSi»­ô*·HùüŽ]
-ꎀ5›

cluster/services/monitoring/tracing.nix

@@ -1,14 +1,16 @@
 { cluster, config, pkgs, ... }:
 
 let
-  inherit (cluster.config) links;
+  inherit (cluster.config.links) prometheus-ingest;
+  inherit (config.links) tempo-grpc;
+  links = cluster.config.hostLinks.${config.networking.hostName};
   dataDir = "/srv/storage/private/tempo";
   tempoConfig = {
     server = {
       http_listen_address = links.tempo.ipv4;
       http_listen_port = links.tempo.port;
-      grpc_listen_address = links.tempo-grpc.ipv4;
-      grpc_listen_port = links.tempo-grpc.port;
+      grpc_listen_address = tempo-grpc.ipv4;
+      grpc_listen_port = tempo-grpc.port;
     };
     distributor.receivers = {
       otlp = {
@@ -19,7 +21,7 @@ let
       };
       zipkin.endpoint = links.tempo-zipkin-http.tuple;
     };
-    querier.frontend_worker.frontend_address = links.tempo-grpc.tuple;
+    querier.frontend_worker.frontend_address = tempo-grpc.tuple;
     ingester = {
       trace_idle_period = "30s";
       max_block_bytes = 1000000;
@@ -56,7 +58,7 @@ let
         path = "${dataDir}/generator/wal";
         remote_write = [
           {
-            url = "${links.prometheus-ingest.url}/api/v1/write";
+            url = "${prometheus-ingest.url}/api/v1/write";
             send_exemplars = true;
           }
         ];
@@ -68,7 +70,11 @@ let
     ];
   };
 in {
-  age.secrets.tempoSecrets.file = ./secrets/tempo-secrets.age;
+  links.tempo-grpc.protocol = "http";
+
+  services.locksmith.waitForSecrets.tempo = [
+    "garage-tempo-ingest"
+  ];
 
   users.users.tempo = {
     isSystemUser = true;
@@ -81,24 +87,53 @@ in {
 
   systemd.services.tempo = {
     wantedBy = [ "multi-user.target" ];
+    distributed = {
+      enable = true;
+      registerServices = [
+        "tempo"
+        "tempo-ingest-otlp-grpc"
+      ];
+    };
     serviceConfig = {
       User = "tempo";
       Group = "tempo";
       ExecStart = "${pkgs.tempo}/bin/tempo -config.file=${pkgs.writeText "tempo.yaml" (builtins.toJSON tempoConfig)}";
       PrivateTmp = true;
-      EnvironmentFile = config.age.secrets.tempoSecrets.path;
+      EnvironmentFile = "/run/locksmith/garage-tempo-ingest";
     };
   };
-  services.grafana.provision.datasources.settings.datasources = [
-    {
-      name = "Tempo";
-      uid = "P214B5B846CF3925F";
-      inherit (links.tempo) url;
-      type = "tempo";
-      jsonData = {
-        serviceMap.datasourceUid = "PBFA97CFB590B2093"; # prometheus
-        nodeGraph.enabled = true;
+
+  consul.services = {
+    tempo = {
+      mode = "manual";
+      definition = {
+        name = "tempo";
+        address = links.tempo.ipv4;
+        inherit (links.tempo) port;
+        checks = [
+          {
+            name = "Tempo";
+            id = "service:tempo:backend";
+            interval = "5s";
+            http = "${links.tempo.url}/ready";
+          }
+        ];
       };
-    }
-  ];
+    };
+    tempo-ingest-otlp-grpc = {
+      mode = "manual";
+      definition = {
+        name = "tempo-ingest-otlp-grpc";
+        address = links.tempo-otlp-grpc.ipv4;
+        inherit (links.tempo-otlp-grpc) port;
+        checks = [
+          {
+            name = "Tempo Service Status";
+            id = "service:tempo-ingest-otlp-grpc:tempo";
+            alias_service = "tempo";
+          }
+        ];
+      };
+    };
+  };
 }

cluster/services/sso/host.nix

@@ -54,7 +54,7 @@ in
       "-Dotel.traces.exporter=otlp"
     ];
     OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
-    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
+    OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.ways.ingest-traces-otlp.url;
     OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
     OTEL_TRACES_SAMPLER_ARG = "0.50";
   };

cluster/services/ways/host.nix

@@ -25,7 +25,14 @@ in
             ];
             locations = lib.mkMerge [
               {
-                "/".proxyPass = cfg.target;
+                "/" = if cfg.grpc then {
+                  extraConfig = ''
+                    set $nix_proxy_grpc_target ${cfg.target};
+                    grpc_pass $nix_proxy_grpc_target;
+                  '';
+                } else {
+                  proxyPass = cfg.target;
+                };
                 "${cfg.healthCheckPath}".extraConfig = "access_log off;";
               }
               {

cluster/services/ways/options/way.nix

@@ -35,6 +35,12 @@ with lib;
       };
     };
 
+    grpc = mkOption {
+      description = "Whether this endpoint is a gRPC service.";
+      type = types.bool;
+      default = false;
+    };
+
     target = mkOption {
       type = types.str;
     };
@@ -101,7 +107,7 @@ with lib;
     (lib.mkIf options.consulService.isDefined {
       useConsul = true;
       nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
-      target = "http://${options.nginxUpstreamName.value}";
+      target = "${if config.grpc then "grpc" else "http"}://${options.nginxUpstreamName.value}";
     })
     (lib.mkIf options.bucket.isDefined {
       consulService = "garage-web";

modules/consul-distributed-services/default.nix

@@ -26,11 +26,13 @@ in
 
       cfg = v.distributed;
 
-      svc = config.consul.services.${cfg.registerService};
+      svcs = map (x: config.consul.services.${x}) cfg.registerServices;
 
       runWithRegistration = pkgs.writeShellScript "run-with-registration" ''
-        trap '${svc.commands.deregister}' EXIT
-        ${svc.commands.register}
+        trap '${lib.concatStringsSep ";" (map (svc: svc.commands.deregister) svcs)}' EXIT
+        ${lib.concatStringsSep "\n" (
+          map (svc: svc.commands.register) svcs
+        )}
         ''${@}
       '';
 
@@ -49,10 +51,10 @@ in
       [Service]
       ExecStartPre=${waitForConsul} 'services/${n}%i'
       ExecStart=
-      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
+      ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerServices != []) runWithRegistration} ${ExecStart}
       Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
       ${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
-      ${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
+      ${optionalString (cfg.registerServices != []) (lib.concatStringsSep "\n" (map (svc: "ExecStopPost=${svc.commands.deregister}") svcs))}
     ''))
   ];
 }

modules/systemd-extras/distributed.nix

@@ -17,6 +17,11 @@ with lib;
           type = with types; nullOr str;
           default = null;
         };
+        registerServices = mkOption {
+          description = "Consul services to register when this service gets started.";
+          type = with types; listOf str;
+          default = if config.distributed.registerService == null then [ ] else [ config.distributed.registerService ];
+        };
       };
     }));
   };