privatevoid.net/depot
1
1
Fork 0
Code Issues 18 Pull requests 2 Projects 3 Releases Packages Wiki Activity Actions

cluster/services/monitoring: make loki HA

Browse source
This commit is contained in:
Max Headroom 2024-08-03 00:56:13 +02:00
parent 201f07efc3
commit 7ca4cead09
4 changed files with 18 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cluster/services/monitoring/client.nix
View file

@ -26,7 +26,7 @@ in {
name = "logging"; name = "logging";
positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml"; positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
clients = singleton { clients = singleton {
url = "${cluster.config.links.loki-ingest.url}/loki/api/v1/push"; url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
}; };
scrape_configs = singleton { scrape_configs = singleton {
job_name = "journal"; job_name = "journal";

14
cluster/services/monitoring/default.nix
View file

@ -14,14 +14,6 @@ in
]; ];
links = { links = {
loki-ingest = {
protocol = "http";
ipv4 = meshIpFor "logging";
};
loki = {
protocol = "http";
ipv4 = meshIpFor "logging";
};
prometheus-ingest = { prometheus-ingest = {
protocol = "http"; protocol = "http";
ipv4 = meshIpFor "server"; ipv4 = meshIpFor "server";
@ -58,7 +50,7 @@ in
client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ]; client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
blackbox = [ "checkmate" "grail" "prophet" ]; blackbox = [ "checkmate" "grail" "prophet" ];
grafana = [ "VEGAS" "prophet" ]; grafana = [ "VEGAS" "prophet" ];
logging = [ "VEGAS" ]; logging = [ "VEGAS" "grail" ];
server = [ "VEGAS" ]; server = [ "VEGAS" ];
}; };
nixos = { nixos = {
@ -74,6 +66,10 @@ in
./tracing.nix ./tracing.nix
]; ];
}; };
meshLinks.logging = {
name = "loki";
link.protocol = "http";
};
}; };
garage = { garage = {

2
cluster/services/monitoring/grafana-ha.nix
View file

@ -2,7 +2,7 @@
let let
inherit (depot.lib.meta) domain; inherit (depot.lib.meta) domain;
inherit (cluster.config.links) loki-ingest prometheus-ingest; inherit (cluster.config.links) prometheus-ingest;
inherit (cluster.config) hostLinks; inherit (cluster.config) hostLinks;

17
cluster/services/monitoring/logging.nix
View file

@ -3,13 +3,17 @@
let let
inherit (config.links) loki-grpc; inherit (config.links) loki-grpc;
inherit (cluster.config.links) loki-ingest; link = cluster.config.hostLinks.${config.networking.hostName}.loki;
cfg = config.services.loki; cfg = config.services.loki;
in in
{ {
links.loki-grpc.protocol = "grpc"; links.loki-grpc.protocol = "grpc";
systemd.services.loki = { systemd.services.loki = {
distributed = {
enable = true;
registerService = "loki";
};
after = [ "wireguard-wgmesh.service" ]; after = [ "wireguard-wgmesh.service" ];
serviceConfig.EnvironmentFile = "/run/locksmith/garage-loki-ingest"; serviceConfig.EnvironmentFile = "/run/locksmith/garage-loki-ingest";
}; };
@ -26,8 +30,8 @@ in
auth_enabled = false; auth_enabled = false;
server = { server = {
log_level = "warn"; log_level = "warn";
http_listen_address = loki-ingest.ipv4; http_listen_address = link.ipv4;
http_listen_port = loki-ingest.port; http_listen_port = link.port;
grpc_listen_address = loki-grpc.ipv4; grpc_listen_address = loki-grpc.ipv4;
grpc_listen_port = loki-grpc.port; grpc_listen_port = loki-grpc.port;
}; };
@ -110,16 +114,17 @@ in
}; };
consul.services.loki = { consul.services.loki = {
mode = "manual";
definition = { definition = {
name = "loki"; name = "loki";
address = loki-ingest.ipv4; address = link.ipv4;
inherit (loki-ingest) port; inherit (link) port;
checks = [ checks = [
{ {
name = "Loki"; name = "Loki";
id = "service:loki:backend"; id = "service:loki:backend";
interval = "5s"; interval = "5s";
http = "${loki-ingest.url}/ready"; http = "${link.url}/ready";
} }
]; ];
}; };