Merge pull request #18 from privatevoid-net/monitoring

Project: Monitoring

hosts/VEGAS/modules/nginx/default.nix

@@ -23,6 +23,8 @@ in
       server_names_hash_bucket_size 128;
       proxy_headers_hash_max_size 4096;
       proxy_headers_hash_bucket_size 128;
+      log_format fmt_loki 'host=$host remote_addr=$remote_addr remote_user=$remote_user request="$request" status=$status body_bytes_sent=$body_bytes_sent http_referer="$http_referer" http_user_agent="$http_user_agent"';
+      access_log syslog:server=unix:/dev/log,tag=nginx_access,nohostname fmt_loki;
     '';
   };
   services.phpfpm.pools.www = {

hosts/VEGAS/services/ipfs/default.nix

@@ -1,4 +1,4 @@
-{ aspect, config, lib, pkgs, tools, ... }:
+{ aspect, config, hosts, lib, pkgs, tools, ... }:
 with tools.nginx;
 let
   inherit (tools.meta) domain;
@@ -56,6 +56,15 @@ in
       locations."/api".proxyPass = "http://unix:/run/ipfs/ipfs-api.sock:";
       locations."/ipns/webui.ipfs.${domain}".proxyPass = "http://127.0.0.1:${gwPort}/ipns/webui.ipfs.${domain}";
       locations."= /".return = "302 /ipns/webui.ipfs.${domain}";
+      locations."/debug/metrics/prometheus" = {
+        proxyPass = "http://unix:/run/ipfs/ipfs-api.sock:";
+        extraConfig = ''
+          access_log off;
+          auth_request off;
+          allow ${hosts.VEGAS.interfaces.primary.addr};
+          deny all;
+        '';
+      };
     };
   };
   services.oauth2_proxy.nginx.virtualHosts = [ "ipfs.admin.${domain}" ];

hosts/VEGAS/services/monitoring/default.nix

@@ -0,0 +1,171 @@
+{ config, hosts, lib, tools, ... }:
+let
+  inherit (tools.meta) domain;
+
+  inherit (config) ports portsStr;
+
+  cfg = { inherit (config.services) loki; };
+
+  toString' = v:
+    if v == true then "true" else
+    if v == false then "false" else
+    toString v;
+
+  mapPaths = lib.mapAttrsRecursive (
+    path: value: lib.nameValuePair
+      (lib.toUpper (lib.concatStringsSep "_" path))
+      (toString' value)
+  );
+
+  translateConfig = config: lib.listToAttrs (
+    lib.collect
+      (x: x ? name && x ? value)
+      (mapPaths config)
+  );
+
+  login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}";
+
+  filteredHosts = lib.filterAttrs (_: host: host ? hypr && host ? nixos) hosts;
+
+  myNode = hosts.${config.networking.hostName};
+in
+{
+  age.secrets.grafana-secrets = {
+    file = ../../../../secrets/grafana-secrets.age;
+  };
+
+  reservePortsFor = [ "grafana" "prometheus" "loki" "loki-grpc" ];
+  services.grafana = {
+    enable = true;
+    port = ports.grafana;
+    rootUrl = "https://monitoring.${domain}/";
+    dataDir = "/srv/storage/private/grafana";
+    analytics.reporting.enable = false;
+    extraOptions = translateConfig {
+      auth.generic_oauth = {
+        enabled = true;
+        allow_sign_up = true;
+        client_id = "net.privatevoid.monitoring1";
+        auth_url = login "auth";
+        token_url = login "token";
+        api_url = login "userinfo";
+        scopes = [ "openid" "profile" "email" "roles" ];
+        role_attribute_strict = true;
+        role_attribute_path = "resource_access.monitoring.roles[0]";
+      };
+      security = {
+        cookie_secure = true;
+        disable_gravatar = true;
+      };
+    };
+    provision = {
+      enable = true;
+      datasources = [
+        {
+          name = "Prometheus";
+          url = "http://127.0.0.1:${portsStr.prometheus}";
+          type = "prometheus";
+          isDefault = true;
+        }
+        {
+          name = "Loki";
+          url = "http://${myNode.hypr.addr}:${portsStr.loki}";
+          type = "loki";
+        }
+      ];
+    };
+  };
+
+  systemd.services.grafana.serviceConfig = {
+    EnvironmentFile = config.age.secrets.grafana-secrets.path;
+  };
+
+  services.nginx.virtualHosts."monitoring.${domain}" = lib.recursiveUpdate (tools.nginx.vhosts.proxy "http://127.0.0.1:${portsStr.grafana}") {
+    locations."/".proxyWebsockets = true;
+  };
+
+  services.prometheus = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = ports.prometheus;
+    globalConfig = {
+      scrape_interval = "60s";
+    };
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = lib.flip lib.mapAttrsToList filteredHosts (name: host: {
+          targets = [ "${host.hypr.addr}:9100" ];
+          labels.instance = name;
+        });
+      }
+      {
+        job_name = "jitsi";
+        static_configs = [
+          {
+            targets = [ "${hosts.prophet.hypr.addr}:9700" ];
+            labels.instance = "meet.${domain}";
+          }
+        ];
+      }
+        {
+          job_name = "ipfs";
+          scheme = "https";
+          metrics_path = "/debug/metrics/prometheus";
+          static_configs = [
+            {
+              targets = [ "ipfs.admin.${domain}" ];
+              labels.instance = "VEGAS";
+            }
+          ];
+        }
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+    dataDir = "/srv/storage/private/loki";
+    configuration = {
+      auth_enabled = false;
+      server = {
+        http_listen_address = myNode.hypr.addr;
+        http_listen_port = ports.loki;
+        grpc_listen_address = "127.0.0.1";
+        grpc_listen_port = ports.loki-grpc;
+      };
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore.store = "inmemory";
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "5m";
+        chunk_retain_period = "30s";
+      };
+      schema_config.configs = [
+        {
+          from = "2022-05-14";
+          store = "boltdb";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "168h";
+          };
+        }
+      ];
+      storage_config = {
+        boltdb.directory = "${cfg.loki.dataDir}/boltdb-index";
+        filesystem.directory = "${cfg.loki.dataDir}/storage-chunks";
+      };
+      limits_config = {
+        enforce_metric_name = false;
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+    };
+  };
+}

hosts/VEGAS/system.nix

@@ -31,6 +31,7 @@
       ./services/mail
       ./services/matrix
       ./services/minecraft
+      ./services/monitoring
       ./services/nix/binary-cache.nix
       ./services/nix/nar-serve.nix
       ./services/object-storage

hosts/prophet/modules/nginx/default.nix

@@ -23,6 +23,8 @@ in
       server_names_hash_bucket_size 128;
       proxy_headers_hash_max_size 4096;
       proxy_headers_hash_bucket_size 128;
+      log_format fmt_loki 'host=$host remote_addr=$remote_addr remote_user=$remote_user request="$request" status=$status body_bytes_sent=$body_bytes_sent http_referer="$http_referer" http_user_agent="$http_user_agent"';
+      access_log syslog:server=unix:/dev/log,tag=nginx_access,nohostname fmt_loki;
     '';
   };
   services.phpfpm.pools.www = {

hosts/prophet/services/meet/default.nix

@@ -17,8 +17,15 @@ in
   };
   services.jitsi-videobridge = {
     openFirewall = true;
-    config.videobridge.ice = {
+    apis = [ "colibri" "rest" ];
-      tcp.port = 7777;
+    config.videobridge = {
+      ice = {
+        tcp.port = 7777;
+      };
+      stats.transports = [
+        { type = "muc"; }
+        { type = "colibri"; }
+      ];
     };
     nat = lib.optionalAttrs isNAT {
       localAddress = interfaces.primary.addr;

modules/default.nix

@@ -11,6 +11,7 @@ let
     hyprspace = import ./hyprspace;
     ipfs = import ./ipfs;
     maintenance = import ./maintenance;
+    monitoring = import ./monitoring;
     nix-builder = import ./nix-builder;
     nix-config = import ./nix-config;
     nix-config-server = import ./nix-config/server.nix;
@@ -38,6 +39,7 @@ in rec {
     server = [
       deploy-rs-receiver
       fail2ban
+      monitoring
       nix-config-server
       system-recovery
     ] ++ base ++ networking;

modules/monitoring/default.nix

@@ -0,0 +1,63 @@
+{ config, hosts, lib, pkgs, ... }:
+let
+  myNode = hosts.${config.networking.hostName};
+
+  writeJSON = filename: data: pkgs.writeText filename (builtins.toJSON data);
+
+  inherit (config) ports portsStr;
+
+  relabel = from: to: {
+    source_labels = [ from ];
+    target_label = to;
+  };
+in
+{
+  # same as remote loki port
+  reservePortsFor = [ "loki" ];
+
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      listenAddress = myNode.hypr.addr;
+    };
+
+    jitsi = {
+      enable = config.services.jitsi-meet.enable;
+      listenAddress = myNode.hypr.addr;
+      interval = "60s";
+    };
+  };
+
+  systemd.services.promtail = {
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      ExecStart = "${pkgs.grafana-loki}/bin/promtail --config.expand-env=true --config.file ${writeJSON "promtail.yaml" {
+        server.disable = true;
+        positions.filename = "\${STATE_DIRECTORY:/tmp}/promtail-positions.yaml";
+        clients = [
+          { url = "http://${hosts.VEGAS.hypr.addr}:${portsStr.loki}/loki/api/v1/push"; }
+        ];
+        scrape_configs = [
+          {
+            job_name = "journal";
+            journal = {
+              max_age = "12h";
+              labels.host = config.networking.hostName;
+            };
+              relabel_configs = [
+                (relabel "__journal__systemd_unit" "systemd_unit")
+                (relabel "__journal__hostname" "machine_name")
+                (relabel "__journal__exe" "executable")
+                (relabel "__journal__comm" "command")
+                (relabel "__journal__boot_id" "systemd_boot_id")
+                (relabel "__journal__systemd_cgroup" "systemd_cgroup")
+                (relabel "__journal_syslog_identifier" "syslog_identifier")
+              ];
+          }
+        ];
+      }}";
+      StateDirectory = "promtail";
+    };
+  };
+}

secrets/grafana-secrets.age

@@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A 2qivuJRscNgO3c+hS8ZkTLdWGKsswTt8qmxM9Uhyixw
+5JbWwwzRFK9uc/6BDgQyWo+6vinZ0E3jf3Bk8nao6Rk
+-> ssh-ed25519 5/zT0w h4WIKhb3AU4FSLr0qLbhK0oLfk531cihqz0IVcXLbFU
+weLZXuXV/A5II8ZP5hzCbhwN1IT0eAlBhKFHlCHaGks
+-> ssh-ed25519 d3WGuA jejKXO17+U/JEdNZQW/0XfUOo3IxH3Di+5gUUCN4zAQ
+5mwkp9+UzTiOWjE+X70egHPU13Iy4xoZS4t/vpO9cE4
+-> C%-grease 6,+fR
+0CJQ4acXn9gqgkyG9B6DG8+VotwqXw
+--- TFou/6wFh0Fcs5KNETBdIKgVxN1EpLufRLADO6vEWyI
+ôçNGDK™©¦¥²“™<E2809C>©§°@o¯nËBÄŠ¨/—Z³§CÜÇœsÅî3µ÷'·jzb§v¸ÓŨõÊ›	µZZžS<C5BE>"-p´<70>E-®¯<çÖ-¤UU)Y½½
+y¾éØW¨iSN ‰

secrets/secrets.nix

@@ -14,6 +14,7 @@ in with hosts;
   "gitlab-secret-jws.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "gitlab-secret-otp.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "gitlab-secret-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "hci-cache-config.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "hci-cache-credentials-prophet.age".publicKeys = max ++ map systemKeys [ prophet ];
   "hci-cache-credentials-VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ];