From 7c72acee59777048996897c237d24f29f61fb96f Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 15 May 2022 00:34:09 +0200 Subject: [PATCH 1/8] hosts/VEGAS: add monitoring service --- hosts/VEGAS/services/monitoring/default.nix | 102 ++++++++++++++++++++ hosts/VEGAS/system.nix | 1 + secrets/grafana-secrets.age | 12 +++ secrets/secrets.nix | 1 + 4 files changed, 116 insertions(+) create mode 100644 hosts/VEGAS/services/monitoring/default.nix create mode 100644 secrets/grafana-secrets.age diff --git a/hosts/VEGAS/services/monitoring/default.nix b/hosts/VEGAS/services/monitoring/default.nix new file mode 100644 index 0000000..16d4317 --- /dev/null +++ b/hosts/VEGAS/services/monitoring/default.nix @@ -0,0 +1,102 @@ +{ config, hosts, lib, tools, ... }: +let + inherit (tools.meta) domain; + + inherit (config) ports portsStr; + + toString' = v: + if v == true then "true" else + if v == false then "false" else + toString v; + + mapPaths = lib.mapAttrsRecursive ( + path: value: lib.nameValuePair + (lib.toUpper (lib.concatStringsSep "_" path)) + (toString' value) + ); + + translateConfig = config: lib.listToAttrs ( + lib.collect + (x: x ? name && x ? value) + (mapPaths config) + ); + + login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}"; + + filteredHosts = lib.filterAttrs (_: host: host ? hypr && host ? nixos) hosts; +in +{ + age.secrets.grafana-secrets = { + file = ../../../../secrets/grafana-secrets.age; + }; + + reservePortsFor = [ "grafana" "prometheus" ]; + services.grafana = { + enable = true; + port = ports.grafana; + rootUrl = "https://monitoring.${domain}/"; + dataDir = "/srv/storage/private/grafana"; + analytics.reporting.enable = false; + extraOptions = translateConfig { + auth.generic_oauth = { + enabled = true; + allow_sign_up = true; + client_id = "net.privatevoid.monitoring1"; + auth_url = login "auth"; + token_url = login "token"; + api_url = login "userinfo"; + scopes = [ "openid" "profile" "email" "roles" ]; + role_attribute_strict = true; + role_attribute_path = "resource_access.monitoring.roles[0]"; + }; + security = { + cookie_secure = true; + disable_gravatar = true; + }; + }; + provision = { + enable = true; + datasources = [ + { + name = "Prometheus"; + url = "http://127.0.0.1:${portsStr.prometheus}"; + type = "prometheus"; + isDefault = true; + } + ]; + }; + }; + + systemd.services.grafana.serviceConfig = { + EnvironmentFile = config.age.secrets.grafana-secrets.path; + }; + + services.nginx.virtualHosts."monitoring.${domain}" = tools.nginx.vhosts.proxy "http://127.0.0.1:${portsStr.grafana}"; + + services.prometheus = { + enable = true; + listenAddress = "127.0.0.1"; + port = ports.prometheus; + globalConfig = { + scrape_interval = "60s"; + }; + scrapeConfigs = [ + { + job_name = "node"; + static_configs = lib.flip lib.mapAttrsToList filteredHosts (name: host: { + targets = [ "${host.hypr.addr}:9100" ]; + labels.instance = name; + }); + } + { + job_name = "jitsi"; + static_configs = [ + { + targets = [ "${hosts.prophet.hypr.addr}:9700" ]; + labels.instance = "meet.${domain}"; + } + ]; + } + ]; + }; +} diff --git a/hosts/VEGAS/system.nix b/hosts/VEGAS/system.nix index 7b911a0..411f0ed 100644 --- a/hosts/VEGAS/system.nix +++ b/hosts/VEGAS/system.nix @@ -31,6 +31,7 @@ ./services/mail ./services/matrix ./services/minecraft + ./services/monitoring ./services/nix/binary-cache.nix ./services/nix/nar-serve.nix ./services/object-storage diff --git a/secrets/grafana-secrets.age b/secrets/grafana-secrets.age new file mode 100644 index 0000000..c96ec1f --- /dev/null +++ b/secrets/grafana-secrets.age @@ -0,0 +1,12 @@ +age-encryption.org/v1 +-> ssh-ed25519 NO562A 2qivuJRscNgO3c+hS8ZkTLdWGKsswTt8qmxM9Uhyixw +5JbWwwzRFK9uc/6BDgQyWo+6vinZ0E3jf3Bk8nao6Rk +-> ssh-ed25519 5/zT0w h4WIKhb3AU4FSLr0qLbhK0oLfk531cihqz0IVcXLbFU +weLZXuXV/A5II8ZP5hzCbhwN1IT0eAlBhKFHlCHaGks +-> ssh-ed25519 d3WGuA jejKXO17+U/JEdNZQW/0XfUOo3IxH3Di+5gUUCN4zAQ +5mwkp9+UzTiOWjE+X70egHPU13Iy4xoZS4t/vpO9cE4 +-> C%-grease 6,+fR +0CJQ4acXn9gqgkyG9B6DG8+VotwqXw +--- TFou/6wFh0Fcs5KNETBdIKgVxN1EpLufRLADO6vEWyI +NGDK@onBĊ/ZCǜs3'jzbvŨʛ ZZS"-pE-<-UU)Y +yWiSN  \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index b7b8220..f0fe63a 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -14,6 +14,7 @@ in with hosts; "gitlab-secret-jws.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "gitlab-secret-otp.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "gitlab-secret-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ]; + "grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "hci-cache-config.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ]; "hci-cache-credentials-prophet.age".publicKeys = max ++ map systemKeys [ prophet ]; "hci-cache-credentials-VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ]; From af127f70bf856e57b0f48a00273e79fce235787a Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 15 May 2022 00:34:45 +0200 Subject: [PATCH 2/8] modules/monitoring: init and add to server group --- modules/default.nix | 2 ++ modules/monitoring/default.nix | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 modules/monitoring/default.nix diff --git a/modules/default.nix b/modules/default.nix index dcfb44a..930da19 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -11,6 +11,7 @@ let hyprspace = import ./hyprspace; ipfs = import ./ipfs; maintenance = import ./maintenance; + monitoring = import ./monitoring; nix-builder = import ./nix-builder; nix-config = import ./nix-config; nix-config-server = import ./nix-config/server.nix; @@ -38,6 +39,7 @@ in rec { server = [ deploy-rs-receiver fail2ban + monitoring nix-config-server system-recovery ] ++ base ++ networking; diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix new file mode 100644 index 0000000..34a0c45 --- /dev/null +++ b/modules/monitoring/default.nix @@ -0,0 +1,18 @@ +{ config, hosts, ... }: +let + myNode = hosts.${config.networking.hostName}; +in +{ + services.prometheus.exporters = { + node = { + enable = true; + listenAddress = myNode.hypr.addr; + }; + + jitsi = { + enable = config.services.jitsi-meet.enable; + listenAddress = myNode.hypr.addr; + interval = "60s"; + }; + }; +} From 243b91babf878f5caed6c29fbf3c9307f29244af Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 15 May 2022 00:35:02 +0200 Subject: [PATCH 3/8] services/meet: enable monitoring --- hosts/prophet/services/meet/default.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hosts/prophet/services/meet/default.nix b/hosts/prophet/services/meet/default.nix index 3dfab1d..c7ae182 100644 --- a/hosts/prophet/services/meet/default.nix +++ b/hosts/prophet/services/meet/default.nix @@ -17,8 +17,15 @@ in }; services.jitsi-videobridge = { openFirewall = true; - config.videobridge.ice = { - tcp.port = 7777; + apis = [ "colibri" "rest" ]; + config.videobridge = { + ice = { + tcp.port = 7777; + }; + stats.transports = [ + { type = "muc"; } + { type = "colibri"; } + ]; }; nat = lib.optionalAttrs isNAT { localAddress = interfaces.primary.addr; From 0249ecdcc322d4a8987d3e5582e325d561a3fd13 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 16 May 2022 00:09:07 +0200 Subject: [PATCH 4/8] services/monitoring: add Loki --- hosts/VEGAS/services/monitoring/default.nix | 58 ++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/hosts/VEGAS/services/monitoring/default.nix b/hosts/VEGAS/services/monitoring/default.nix index 16d4317..0636103 100644 --- a/hosts/VEGAS/services/monitoring/default.nix +++ b/hosts/VEGAS/services/monitoring/default.nix @@ -4,6 +4,8 @@ let inherit (config) ports portsStr; + cfg = { inherit (config.services) loki; }; + toString' = v: if v == true then "true" else if v == false then "false" else @@ -24,13 +26,15 @@ let login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}"; filteredHosts = lib.filterAttrs (_: host: host ? hypr && host ? nixos) hosts; + + myNode = hosts.${config.networking.hostName}; in { age.secrets.grafana-secrets = { file = ../../../../secrets/grafana-secrets.age; }; - reservePortsFor = [ "grafana" "prometheus" ]; + reservePortsFor = [ "grafana" "prometheus" "loki" "loki-grpc" ]; services.grafana = { enable = true; port = ports.grafana; @@ -63,6 +67,11 @@ in type = "prometheus"; isDefault = true; } + { + name = "Loki"; + url = "http://${myNode.hypr.addr}:${portsStr.loki}"; + type = "loki"; + } ]; }; }; @@ -99,4 +108,51 @@ in } ]; }; + + services.loki = { + enable = true; + dataDir = "/srv/storage/private/loki"; + configuration = { + auth_enabled = false; + server = { + http_listen_address = myNode.hypr.addr; + http_listen_port = ports.loki; + grpc_listen_address = "127.0.0.1"; + grpc_listen_port = ports.loki-grpc; + }; + ingester = { + lifecycler = { + address = "127.0.0.1"; + ring = { + kvstore.store = "inmemory"; + replication_factor = 1; + }; + final_sleep = "0s"; + }; + chunk_idle_period = "5m"; + chunk_retain_period = "30s"; + }; + schema_config.configs = [ + { + from = "2022-05-14"; + store = "boltdb"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "168h"; + }; + } + ]; + storage_config = { + boltdb.directory = "${cfg.loki.dataDir}/boltdb-index"; + filesystem.directory = "${cfg.loki.dataDir}/storage-chunks"; + }; + limits_config = { + enforce_metric_name = false; + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + }; + }; + }; } From 563efc95e40dffef20a63174cb8ce398ef5d8c27 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 16 May 2022 00:09:41 +0200 Subject: [PATCH 5/8] modules/monitoring: ship journal logs to Loki --- modules/monitoring/default.nix | 47 +++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix index 34a0c45..46e7ba9 100644 --- a/modules/monitoring/default.nix +++ b/modules/monitoring/default.nix @@ -1,8 +1,20 @@ -{ config, hosts, ... }: +{ config, hosts, lib, pkgs, ... }: let myNode = hosts.${config.networking.hostName}; + + writeJSON = filename: data: pkgs.writeText filename (builtins.toJSON data); + + inherit (config) ports portsStr; + + relabel = from: to: { + source_labels = [ from ]; + target_label = to; + }; in { + # same as remote loki port + reservePortsFor = [ "loki" ]; + services.prometheus.exporters = { node = { enable = true; @@ -15,4 +27,37 @@ in interval = "60s"; }; }; + + systemd.services.promtail = { + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + ExecStart = "${pkgs.grafana-loki}/bin/promtail --config.expand-env=true --config.file ${writeJSON "promtail.yaml" { + server.disable = true; + positions.filename = "\${STATE_DIRECTORY:/tmp}/promtail-positions.yaml"; + clients = [ + { url = "http://${hosts.VEGAS.hypr.addr}:${portsStr.loki}/loki/api/v1/push"; } + ]; + scrape_configs = [ + { + job_name = "journal"; + journal = { + max_age = "12h"; + labels.host = config.networking.hostName; + }; + relabel_configs = [ + (relabel "__journal__systemd_unit" "systemd_unit") + (relabel "__journal__hostname" "machine_name") + (relabel "__journal__exe" "executable") + (relabel "__journal__comm" "command") + (relabel "__journal__boot_id" "systemd_boot_id") + (relabel "__journal__systemd_cgroup" "systemd_cgroup") + (relabel "__journal_syslog_identifier" "syslog_identifier") + ]; + } + ]; + }}"; + StateDirectory = "promtail"; + }; + }; } From 1d32eb782e59fb76311d58d33f319be0c551d59b Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 16 May 2022 00:10:14 +0200 Subject: [PATCH 6/8] modules/nginx: log to syslog in logfmt format --- hosts/VEGAS/modules/nginx/default.nix | 2 ++ hosts/prophet/modules/nginx/default.nix | 2 ++ 2 files changed, 4 insertions(+) diff --git a/hosts/VEGAS/modules/nginx/default.nix b/hosts/VEGAS/modules/nginx/default.nix index 7ff77d7..1ab99ae 100644 --- a/hosts/VEGAS/modules/nginx/default.nix +++ b/hosts/VEGAS/modules/nginx/default.nix @@ -23,6 +23,8 @@ in server_names_hash_bucket_size 128; proxy_headers_hash_max_size 4096; proxy_headers_hash_bucket_size 128; + log_format fmt_loki 'host=$host remote_addr=$remote_addr remote_user=$remote_user request="$request" status=$status body_bytes_sent=$body_bytes_sent http_referer="$http_referer" http_user_agent="$http_user_agent"'; + access_log syslog:server=unix:/dev/log,tag=nginx_access,nohostname fmt_loki; ''; }; services.phpfpm.pools.www = { diff --git a/hosts/prophet/modules/nginx/default.nix b/hosts/prophet/modules/nginx/default.nix index 7ff77d7..1ab99ae 100644 --- a/hosts/prophet/modules/nginx/default.nix +++ b/hosts/prophet/modules/nginx/default.nix @@ -23,6 +23,8 @@ in server_names_hash_bucket_size 128; proxy_headers_hash_max_size 4096; proxy_headers_hash_bucket_size 128; + log_format fmt_loki 'host=$host remote_addr=$remote_addr remote_user=$remote_user request="$request" status=$status body_bytes_sent=$body_bytes_sent http_referer="$http_referer" http_user_agent="$http_user_agent"'; + access_log syslog:server=unix:/dev/log,tag=nginx_access,nohostname fmt_loki; ''; }; services.phpfpm.pools.www = { From 0cd40f0f475bdc77f296c417f8c90540b5c25153 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 16 May 2022 00:10:58 +0200 Subject: [PATCH 7/8] VEGAS/ipfs: expose metrics --- hosts/VEGAS/services/ipfs/default.nix | 11 ++++++++++- hosts/VEGAS/services/monitoring/default.nix | 11 +++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/hosts/VEGAS/services/ipfs/default.nix b/hosts/VEGAS/services/ipfs/default.nix index 62aef9c..0ceb2f6 100644 --- a/hosts/VEGAS/services/ipfs/default.nix +++ b/hosts/VEGAS/services/ipfs/default.nix @@ -1,4 +1,4 @@ -{ aspect, config, lib, pkgs, tools, ... }: +{ aspect, config, hosts, lib, pkgs, tools, ... }: with tools.nginx; let inherit (tools.meta) domain; @@ -56,6 +56,15 @@ in locations."/api".proxyPass = "http://unix:/run/ipfs/ipfs-api.sock:"; locations."/ipns/webui.ipfs.${domain}".proxyPass = "http://127.0.0.1:${gwPort}/ipns/webui.ipfs.${domain}"; locations."= /".return = "302 /ipns/webui.ipfs.${domain}"; + locations."/debug/metrics/prometheus" = { + proxyPass = "http://unix:/run/ipfs/ipfs-api.sock:"; + extraConfig = '' + access_log off; + auth_request off; + allow ${hosts.VEGAS.interfaces.primary.addr}; + deny all; + ''; + }; }; }; services.oauth2_proxy.nginx.virtualHosts = [ "ipfs.admin.${domain}" ]; diff --git a/hosts/VEGAS/services/monitoring/default.nix b/hosts/VEGAS/services/monitoring/default.nix index 0636103..dedea69 100644 --- a/hosts/VEGAS/services/monitoring/default.nix +++ b/hosts/VEGAS/services/monitoring/default.nix @@ -106,6 +106,17 @@ in } ]; } + { + job_name = "ipfs"; + scheme = "https"; + metrics_path = "/debug/metrics/prometheus"; + static_configs = [ + { + targets = [ "ipfs.admin.${domain}" ]; + labels.instance = "VEGAS"; + } + ]; + } ]; }; From 917aec5b3ac57a665ea28ded29a6bea81ab49102 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 16 May 2022 00:11:36 +0200 Subject: [PATCH 8/8] services/monitoring: allow WebSocket connections --- hosts/VEGAS/services/monitoring/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hosts/VEGAS/services/monitoring/default.nix b/hosts/VEGAS/services/monitoring/default.nix index dedea69..56a56cf 100644 --- a/hosts/VEGAS/services/monitoring/default.nix +++ b/hosts/VEGAS/services/monitoring/default.nix @@ -80,7 +80,9 @@ in EnvironmentFile = config.age.secrets.grafana-secrets.path; }; - services.nginx.virtualHosts."monitoring.${domain}" = tools.nginx.vhosts.proxy "http://127.0.0.1:${portsStr.grafana}"; + services.nginx.virtualHosts."monitoring.${domain}" = lib.recursiveUpdate (tools.nginx.vhosts.proxy "http://127.0.0.1:${portsStr.grafana}") { + locations."/".proxyWebsockets = true; + }; services.prometheus = { enable = true;