depot/cluster/services/monitoring/client.nix

{ cluster, lib, ... }:
let
  myNode = cluster.config.vars.mesh.${cluster.config.vars.hostName};

  relabel = from: to: {
    source_labels = [ from ];
    target_label = to;
  };
in {
  services.journald.extraConfig = "Storage=volatile";

  services.prometheus.exporters = {
    node = {
      enable = true;
      listenAddress = myNode.meshIp;
      enabledCollectors = [
        "systemd"
      ];
    };
  };

  systemd.services.prometheus-node-exporter = {
    after = [ "wireguard-wgmesh.service" ];
    serviceConfig.RestartSec = "10s";
  };

  services.grafana-agent = {
    enable = true;
    settings = {
      logs.configs = lib.singleton {
        name = "logging";
        positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
        clients = [
          { url = "${cluster.config.links.loki-ingest.url}/loki/api/v1/push"; }
        ];
        scrape_configs = [
          {
            job_name = "journal";
            journal = {
              max_age = "12h";
              labels.host = cluster.config.vars.hostName;
            };
              relabel_configs = [
                (relabel "__journal__systemd_unit" "systemd_unit")
                (relabel "__journal__hostname" "machine_name")
                (relabel "__journal__exe" "executable")
                (relabel "__journal__comm" "command")
                (relabel "__journal__boot_id" "systemd_boot_id")
                (relabel "__journal__systemd_cgroup" "systemd_cgroup")
                (relabel "__journal_syslog_identifier" "syslog_identifier")
              ];
          }
        ];
      };
    };
  };
}