depot/hosts/VEGAS/services/monitoring/default.nix

194 lines
5 KiB
Nix
Raw Normal View History

{ cluster, config, hosts, inputs, lib, pkgs, tools, ... }:
2022-05-15 01:34:09 +03:00
let
inherit (tools.meta) domain;
2022-06-18 03:44:51 +03:00
inherit (config) links;
2022-05-15 01:34:09 +03:00
inherit (cluster.config.links) loki-ingest;
2022-05-16 01:09:07 +03:00
cfg = { inherit (config.services) loki; };
2022-05-15 01:34:09 +03:00
toString' = v:
if v == true then "true" else
if v == false then "false" else
toString v;
mapPaths = lib.mapAttrsRecursive (
path: value: lib.nameValuePair
(lib.toUpper (lib.concatStringsSep "_" path))
(toString' value)
);
translateConfig = config: lib.listToAttrs (
lib.collect
(x: x ? name && x ? value)
(mapPaths config)
);
login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}";
2022-05-16 01:09:07 +03:00
myNode = hosts.${config.networking.hostName};
2022-05-15 01:34:09 +03:00
in
{
imports = [
./tracing.nix
];
2022-05-15 01:34:09 +03:00
age.secrets.grafana-secrets = {
file = ../../../../secrets/grafana-secrets.age;
};
2022-06-18 03:44:51 +03:00
links = {
grafana.protocol = "http";
prometheus.protocol = "http";
loki-grpc = {
protocol = "grpc";
};
};
2022-05-15 01:34:09 +03:00
services.grafana = {
enable = true;
2022-06-18 22:50:27 +03:00
package = inputs.self.packages.${pkgs.system}.grafana;
2022-06-18 03:44:51 +03:00
inherit (links.grafana) port;
2022-05-15 01:34:09 +03:00
rootUrl = "https://monitoring.${domain}/";
dataDir = "/srv/storage/private/grafana";
analytics.reporting.enable = false;
extraOptions = translateConfig {
auth.generic_oauth = {
enabled = true;
allow_sign_up = true;
client_id = "net.privatevoid.monitoring1";
auth_url = login "auth";
token_url = login "token";
api_url = login "userinfo";
scopes = [ "openid" "profile" "email" "roles" ];
role_attribute_strict = true;
role_attribute_path = "resource_access.monitoring.roles[0]";
};
security = {
cookie_secure = true;
disable_gravatar = true;
};
feature_toggles.enable = [
"tempoSearch"
"tempoBackendSearch"
"tempoServiceGraph"
];
2022-05-15 01:34:09 +03:00
};
provision = {
enable = true;
datasources = [
{
name = "Prometheus";
# wait for https://github.com/NixOS/nixpkgs/pull/175330
# uid = "PBFA97CFB590B2093";
2022-06-18 03:44:51 +03:00
inherit (links.prometheus) url;
2022-05-15 01:34:09 +03:00
type = "prometheus";
isDefault = true;
}
2022-05-16 01:09:07 +03:00
{
name = "Loki";
# uid = "P8E80F9AEF21F6940";
inherit (loki-ingest) url;
2022-05-16 01:09:07 +03:00
type = "loki";
}
2022-05-15 01:34:09 +03:00
];
};
};
systemd.services.grafana.serviceConfig = {
EnvironmentFile = config.age.secrets.grafana-secrets.path;
};
2022-06-18 03:44:51 +03:00
services.nginx.virtualHosts."monitoring.${domain}" = lib.recursiveUpdate (tools.nginx.vhosts.proxy links.grafana.url) {
locations."/".proxyWebsockets = true;
};
2022-05-15 01:34:09 +03:00
services.prometheus = {
enable = true;
2022-06-18 03:44:51 +03:00
listenAddress = links.prometheus.ipv4;
inherit (links.prometheus) port;
extraFlags = [ "--enable-feature=remote-write-receiver" ];
2022-05-15 01:34:09 +03:00
globalConfig = {
scrape_interval = "60s";
};
scrapeConfigs = [
{
job_name = "node";
static_configs = lib.flip lib.mapAttrsToList cluster.config.vars.mesh (name: host: {
targets = [ "${host.meshIp}:9100" ];
2022-05-15 01:34:09 +03:00
labels.instance = name;
});
}
{
job_name = "jitsi";
static_configs = [
{
targets = [ "${cluster.config.vars.mesh.prophet.meshIp}:9700" ];
2022-05-15 01:34:09 +03:00
labels.instance = "meet.${domain}";
}
];
}
2022-05-16 01:10:58 +03:00
{
job_name = "ipfs";
scheme = "https";
metrics_path = "/debug/metrics/prometheus";
static_configs = [
{
targets = [ "ipfs.admin.${domain}" ];
labels.instance = "VEGAS";
}
];
}
2022-05-15 01:34:09 +03:00
];
};
2022-05-16 01:09:07 +03:00
systemd.services.loki.after = [ "wireguard-wgmesh.service" ];
2022-05-16 01:09:07 +03:00
services.loki = {
enable = true;
dataDir = "/srv/storage/private/loki";
configuration = {
auth_enabled = false;
server = {
log_level = "warn";
http_listen_address = loki-ingest.ipv4;
http_listen_port = loki-ingest.port;
2022-06-18 03:44:51 +03:00
grpc_listen_address = links.loki-grpc.ipv4;
grpc_listen_port = links.loki-grpc.port;
2022-05-16 01:09:07 +03:00
};
2022-06-18 03:44:51 +03:00
frontend_worker.frontend_address = links.loki-grpc.tuple;
2022-05-16 01:09:07 +03:00
ingester = {
lifecycler = {
address = "127.0.0.1";
ring = {
kvstore.store = "inmemory";
replication_factor = 1;
};
final_sleep = "0s";
};
chunk_idle_period = "5m";
chunk_retain_period = "30s";
};
schema_config.configs = [
{
from = "2022-05-14";
store = "boltdb";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
period = "168h";
};
}
];
storage_config = {
boltdb.directory = "${cfg.loki.dataDir}/boltdb-index";
filesystem.directory = "${cfg.loki.dataDir}/storage-chunks";
};
limits_config = {
enforce_metric_name = false;
reject_old_samples = true;
reject_old_samples_max_age = "168h";
};
};
};
2022-05-15 01:34:09 +03:00
}