cluster/services/monitoring: move tempo to s3

cluster/services/monitoring/secrets/tempo-secrets.age

@@ -0,0 +1,16 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A KhCGp7PAMGrEdzRxBrsW4tRk30JwpI+4lPzrRCUhSw4
+8s7WqA5c3zS1euN5R+jfFNBdvr8OQW8P4NFeqtNsIKo
+-> ssh-ed25519 5/zT0w 79hJQ2H76EZTW7YcQFCtKaS5Kbssx4Z8dPFjIVzRgFk
+A1fDJbUnyIRy+kWa3PhJNj/SdRPlcEy6FYsAfnuZ2AQ
+-> ssh-ed25519 d3WGuA aylkdL1KliM1NfrYDGlG8X6YjXvVUCU4sV90I+a840U
+6sXdqIPjtoNSylZRh1DCghHOwDo+fC7WB4QWQoWmG48
+-> //gd+2-grease baUWA$3 z-qs3W O/2.1W
+Sfq3+rkMJhpUTTmcos5TaaUtX2Ip9pciHAZLiWPix+C9N7ccac/1W5RNedMJCLsq
+MQ+xKzexf8+hgNVhKOksvbKBBROXqk1bUOKk8w3OgFPmmByzmCBUwkdkeu5DFTYR
+rg
+--- kUl1uIPRkM5y7C68kdN22pMKXP7gazyha4PE+ap0Jqw
+w>×Àè¥
+<15>CÈ,\‰ßœI¯ˆúHxG@^Çá“PåÃþÙÏlw6
µŽ{þ’rb‘é5æ†T>Êñ
+ÚWܤX4Kp(ß?9ˆß­^^oP3f </v3N$ê¤sÓbŽ¾>O™÷œ+òN0άïµàDtêŽ5Vº#è ¶³‘Uã îŸ#y|›@ŒGzSi»­ô*·HùüŽ]
+ꎀ5›

cluster/services/monitoring/tracing.nix

@@ -33,10 +33,15 @@ let
       };
     };
     storage.trace = {
-      backend = "local";
+      backend = "s3";
       block.bloom_filter_false_positive = 0.05;
       wal.path = "${dataDir}/wal";
-      local.path = "${dataDir}/blocks";
+      s3 = {
+        bucket = "tempo-chunks";
+        endpoint = cluster.config.links.garageS3.hostname;
+        region = "us-east-1";
+        forcepathstyle = true;
+      };
       pool = {
         max_workers = 16;
         queue_depth = 1000;
@@ -63,6 +68,8 @@ let
     ];
   };
 in {
+  age.secrets.tempoSecrets.file = ./secrets/tempo-secrets.age;
+
   users.users.tempo = {
     isSystemUser = true;
     group = "tempo";
@@ -79,6 +86,7 @@ in {
       Group = "tempo";
       ExecStart = "${pkgs.tempo}/bin/tempo -config.file=${pkgs.writeText "tempo.yaml" (builtins.toJSON tempoConfig)}";
       PrivateTmp = true;
+      EnvironmentFile = config.age.secrets.tempoSecrets.path;
     };
   };
   services.grafana.provision.datasources.settings.datasources = [

secrets.nix

@@ -43,6 +43,7 @@ in with hosts;
   "cluster/services/monitoring/secrets/grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "cluster/services/monitoring/secrets/loki-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/monitoring/secrets/secret-monitoring/blackbox.age".publicKeys = max ++ map systemKeys [ checkmate grail prophet ];
+  "cluster/services/monitoring/secrets/tempo-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/patroni/passwords/metrics.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/patroni/passwords/replication.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/patroni/passwords/rewind.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];