VEGAS/monitoring: move to cluster

cluster/services/monitoring/default.nix

@@ -1,18 +1,35 @@
 { config, ... }:
 
+let
+  nodeFor = nodeType: builtins.head config.services.monitoring.nodes.${nodeType};
+
+  meshIpFor = nodeType: config.vars.mesh.${nodeFor nodeType}.meshIp;
+in
+
 {
   links = {
     loki-ingest = {
       protocol = "http";
-      ipv4 = config.vars.mesh.VEGAS.meshIp;
+      ipv4 = meshIpFor "logging";
+    };
+    loki = {
+      protocol = "http";
+      ipv4 = meshIpFor "logging";
     };
   };
   services.monitoring = {
     nodes = {
       client = [ "checkmate" "thunderskin" "VEGAS" "prophet" ];
+      logging = [ "VEGAS" ];
+      server = [ "VEGAS" ];
     };
     nixos = {
       client = ./client.nix;
+      logging = ./logging.nix;
+      server = [
+        ./server.nix
+        ./tracing.nix
+      ];
     };
   };
 }

cluster/services/monitoring/logging.nix

@@ -0,0 +1,61 @@
+{ config, cluster, ... }:
+
+let
+  inherit (config.links) loki-grpc;
+
+  inherit (cluster.config.links) loki-ingest;
+
+  cfg = config.services.loki;
+in
+{
+  links.loki-grpc.protocol = "grpc";
+  systemd.services.loki.after = [ "wireguard-wgmesh.service" ];
+  services.loki = {
+    enable = true;
+    dataDir = "/srv/storage/private/loki";
+    configuration = {
+      auth_enabled = false;
+      server = {
+        log_level = "warn";
+        http_listen_address = loki-ingest.ipv4;
+        http_listen_port = loki-ingest.port;
+        grpc_listen_address = loki-grpc.ipv4;
+        grpc_listen_port = loki-grpc.port;
+      };
+      frontend_worker.frontend_address = loki-grpc.tuple;
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore.store = "inmemory";
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "5m";
+        chunk_retain_period = "30s";
+      };
+      schema_config.configs = [
+        {
+          from = "2022-05-14";
+          store = "boltdb";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "168h";
+          };
+        }
+      ];
+      storage_config = {
+        boltdb.directory = "${cfg.dataDir}/boltdb-index";
+        filesystem.directory = "${cfg.dataDir}/storage-chunks";
+      };
+      limits_config = {
+        enforce_metric_name = false;
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+    };
+  };
+}

cluster/services/monitoring/server.nix

@@ -6,26 +6,18 @@ let
 
   inherit (cluster.config.links) loki-ingest;
 
-  cfg = { inherit (config.services) loki; };
-
   iniList = lib.concatStringsSep " ";
 
   login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}";
 in
 {
-  imports = [
-    ./tracing.nix
-  ];
   age.secrets.grafana-secrets = {
-    file = ../../../../secrets/grafana-secrets.age;
+    file = ./secrets/grafana-secrets.age;
   };
 
   links = {
     grafana.protocol = "http";
     prometheus.protocol = "http";
-    loki-grpc = {
-      protocol = "grpc";
-    };
   };
   services.grafana = {
     enable = true;
@@ -125,53 +117,4 @@ in
     ];
   };
 
-  systemd.services.loki.after = [ "wireguard-wgmesh.service" ];
-  services.loki = {
-    enable = true;
-    dataDir = "/srv/storage/private/loki";
-    configuration = {
-      auth_enabled = false;
-      server = {
-        log_level = "warn";
-        http_listen_address = loki-ingest.ipv4;
-        http_listen_port = loki-ingest.port;
-        grpc_listen_address = links.loki-grpc.ipv4;
-        grpc_listen_port = links.loki-grpc.port;
-      };
-      frontend_worker.frontend_address = links.loki-grpc.tuple;
-      ingester = {
-        lifecycler = {
-          address = "127.0.0.1";
-          ring = {
-            kvstore.store = "inmemory";
-            replication_factor = 1;
-          };
-          final_sleep = "0s";
-        };
-        chunk_idle_period = "5m";
-        chunk_retain_period = "30s";
-      };
-      schema_config.configs = [
-        {
-          from = "2022-05-14";
-          store = "boltdb";
-          object_store = "filesystem";
-          schema = "v11";
-          index = {
-            prefix = "index_";
-            period = "168h";
-          };
-        }
-      ];
-      storage_config = {
-        boltdb.directory = "${cfg.loki.dataDir}/boltdb-index";
-        filesystem.directory = "${cfg.loki.dataDir}/storage-chunks";
-      };
-      limits_config = {
-        enforce_metric_name = false;
-        reject_old_samples = true;
-        reject_old_samples_max_age = "168h";
-      };
-    };
-  };
 }

hosts/VEGAS/system.nix

@@ -26,7 +26,6 @@
       ./services/mail
       ./services/matrix
       ./services/minecraft
-      ./services/monitoring
       ./services/nix/binary-cache.nix
       ./services/nix/nar-serve.nix
       ./services/object-storage

secrets.nix

@@ -23,6 +23,7 @@ in with hosts;
   "cluster/services/ipfs/cluster-secret.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "cluster/services/ipfs/cluster-pinsvc-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "cluster/services/irc/irc-peer-key.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
+  "cluster/services/monitoring/secrets/grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/patroni/passwords/replication.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/patroni/passwords/rewind.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
@@ -39,7 +40,6 @@ in with hosts;
   "secrets/gitlab-secret-jws.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "secrets/gitlab-secret-otp.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "secrets/gitlab-secret-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ];
-  "secrets/grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "secrets/hydra-bincache.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "secrets/hydra-builder-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "secrets/hydra-db-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS ];