cluster/services/monitoring: add blackbox_exporter

cluster/services/monitoring/blackbox.nix

@@ -0,0 +1,82 @@
+{ config, cluster, lib, tools, ... }:
+
+let
+  inherit (lib) flip pipe mapAttrsToList range recursiveUpdate substring;
+
+  inherit (tools.meta) domain;
+  inherit (cluster.config) vars;
+
+  mapTargets = mapAttrsToList (name: value: value // { name = "default/${name}"; });
+
+  mkSecretTargets = amount: map (flip pipe [
+    toString
+    (num: let
+      prefix = "SECRET_MONITORING_BLACKBOX_TARGET_${num}";
+    in {
+      name = "secret/\${${prefix}_NAME}";
+      module = "\${${prefix}_MODULE}";
+      address = "\${${prefix}_ADDRESS}";
+    })
+  ]) (range 1 1);
+
+  probeId = pipe "blackbox-probe-${domain}-${vars.hostName}" [
+    (builtins.hashString "md5")
+    (substring 0 8)
+  ];
+
+  probeUserAgent = "Private Void Monitoring Probe ${probeId}";
+
+  defaultHttpHeaders = {
+    User-Agent = probeUserAgent;
+  };
+
+  relabel = from: to: {
+    source_labels = [ from ];
+    target_label = to;
+  };
+in
+
+{
+  services.grafana-agent.settings.integrations.blackbox = {
+    enabled = true;
+    instance = vars.hostName;
+    scrape_interval = "600s";
+    relabel_configs = [
+      (relabel "__param_module" "module")
+      (relabel "__param_target" "target")
+      {
+        target_label = "probe_id";
+        replacement = probeId;
+      }
+    ];
+    blackbox_config.modules = rec {
+      http2xx = {
+        prober = "http";
+        http = {
+          headers = defaultHttpHeaders;
+          preferred_ip_protocol = "ip4";
+        };
+      };
+      https2xx = recursiveUpdate http2xx {
+        http.fail_if_not_ssl = true;
+      };
+    };
+    blackbox_targets = let
+      regularTargets = mapTargets {
+        web = {
+          module = "https2xx";
+          address = "https://www.${domain}";
+        };
+      };
+      secretTargets = mkSecretTargets 1;
+    in regularTargets ++ secretTargets;
+  };
+
+  age.secrets = {
+    grafana-agent-blackbox-secret-monitoring.file = ./secrets/secret-monitoring/blackbox.age;
+  };
+
+  systemd.services.grafana-agent.serviceConfig = {
+    EnvironmentFile = config.age.secrets.grafana-agent-blackbox-secret-monitoring.path;
+  };
+}

cluster/services/monitoring/default.nix

@@ -24,11 +24,13 @@ in
   services.monitoring = {
     nodes = {
       client = [ "checkmate" "thunderskin" "VEGAS" "prophet" ];
+      blackbox = [ "checkmate" "VEGAS" "prophet" ];
       logging = [ "VEGAS" ];
       server = [ "VEGAS" ];
     };
     nixos = {
       client = ./client.nix;
+      blackbox = ./blackbox.nix;
       logging = ./logging.nix;
       server = [
         ./server.nix

cluster/services/monitoring/secrets/secret-monitoring/blackbox.age

@@ -0,0 +1,15 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A A0vL6E3vkVkzegk3cv7Vz+xkZeWuigw+/02SK5XmKnA
+jpsf/NNQvuGahVsEs+WmWujM62x3h01yyCD90xLdNtM
+-> ssh-ed25519 5/zT0w Y54Z16HFsY+E2NsCVM6aC2vnSn3AnFpIp86z16Xh+VE
+OmXbCRAJMvkitP5B6NhoEoBd4WJcKM1h6KMrHVAyCIQ
+-> ssh-ed25519 TCgorQ 5LeFvOyvEqr4UDM+1VLlzrQCGU2FVGVYzRIOpX/ZjS0
+YsOx29l4p/NOpTBqDntCsuhonMI8g3OEJ3YMCUCx6wI
+-> ssh-ed25519 d3WGuA 9FrEqFZ68VtIk6aVALx77wrFI+iuY/JJ9C9X9s3+dC0
+o6SJEbSxUWCF4uvqH30qpgK0KVIsW2rwbLNTZWIGiZs
+-> ssh-ed25519 YIaSKQ iiCjEeTuaABSh1ruLy1oPG3nmHGXDgfQIQK0q4EDAH0
+e6SFbfOmqPMueEbDG48AezBgj3QGlAFZEi0OtLToipc
+-> Hs)Ns-grease T+
+8uGnEXk
+--- 0lnf5VB58DXb87+8dvQUegA57JHCGhWQCskxfAeEM1Q
+Ú‡|·PD
L€…Ø9x¾‡ºÍÉôu FàxwÃH¤±Shv­O¬ÖÑÊï7m¼ñJ€z4éöe€<65>ꯜºNsó
¸ý„yÆh‚ÍÄçÔ«ÐS0ï I_°-v¡˜taƒ
æ/Ûýå?öˆzC‡éfp"»—\àÛ>þ8ñWÞ;r
=ãÉZ׬¿N­W_føSq‰Çp<C387>YxHRvÚáQúS‡<53>ãZ)ú!ÈŸî„Ñ|^qÓ}–œvNEæz;[,	'ôc—½­†Û‹àhŸl ³Ä

secrets.nix

@@ -25,6 +25,7 @@ in with hosts;
   "cluster/services/irc/irc-peer-key.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "cluster/services/monitoring/secrets/grafana-db-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/monitoring/secrets/grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
+  "cluster/services/monitoring/secrets/secret-monitoring/blackbox.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
   "cluster/services/patroni/passwords/replication.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/patroni/passwords/rewind.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];