diff --git a/cluster/services/patroni/default.nix b/cluster/services/patroni/default.nix new file mode 100644 index 0000000..7c21e7a --- /dev/null +++ b/cluster/services/patroni/default.nix @@ -0,0 +1,49 @@ +{ config, lib, ... }: + +let + inherit (config.vars) hosts; + + cfg = config.services.patroni; + + renameToLink = mode: n: v: lib.nameValuePair "patroni-etcd-node-${mode}-${n}" v; + + genLinks = mode: nodes: f: lib.mapAttrs' (renameToLink mode) (lib.genAttrs nodes f); + + getMeshIp = name: config.vars.mesh.${name}.meshIp; + + mkLink = name: { + ipv4 = getMeshIp name; + protocol = "http"; + }; +in +{ + vars.patroni = { + etcdNodes = lib.genAttrs cfg.nodes.etcd (name: config.links."patroni-etcd-node-peer-${name}"); + etcdNodesClient = lib.genAttrs cfg.nodes.etcd (name: config.links."patroni-etcd-node-client-${name}"); + etcdExtraNodes = [ "fly=http://10.1.1.151:2380" ]; + passwords = { + PATRONI_REPLICATION_PASSWORD = ./passwords/replication.age; + PATRONI_SUPERUSER_PASSWORD = ./passwords/superuser.age; + PATRONI_REWIND_PASSWORD = ./passwords/rewind.age; + }; + }; + links = genLinks "client" cfg.nodes.etcd mkLink + // genLinks "peer" cfg.nodes.etcd mkLink + // { + patroni-pg-internal.ipv4 = "0.0.0.0"; + patroni-api.ipv4 = "0.0.0.0"; + patroni-pg-access.ipv4 = "127.0.0.1"; + }; + services.patroni = { + nodes = { + worker = [ "VEGAS" "prophet" ]; + etcd = [ "VEGAS" "prophet" ]; + haproxy = [ "VEGAS" "prophet" ]; + }; + nixos = { + worker = ./worker.nix; + etcd = ./etcd.nix; + haproxy = ./haproxy.nix; + }; + }; +} diff --git a/cluster/services/patroni/etcd.nix b/cluster/services/patroni/etcd.nix new file mode 100644 index 0000000..8f1cb5f --- /dev/null +++ b/cluster/services/patroni/etcd.nix @@ -0,0 +1,21 @@ +{ cluster, lib, pkgs, ... }: + +let + inherit (cluster.config) vars; + + getEtcdUrl = name: vars.patroni.etcdNodes.${name}.url; + + mkMember = n: "${n}=${getEtcdUrl n}"; +in + +{ + services.etcd = { + enable = true; + dataDir = "/srv/storage/private/etcd"; + initialCluster = (map mkMember cluster.config.services.patroni.nodes.etcd) ++ vars.patroni.etcdExtraNodes; + listenPeerUrls = lib.singleton vars.patroni.etcdNodes.${vars.hostName}.url; + listenClientUrls = lib.singleton vars.patroni.etcdNodesClient.${vars.hostName}.url; + }; + # run on any architecture + systemd.services.etcd.environment.ETCD_UNSUPPORTED_ARCH = pkgs.go.GOARCH; +} diff --git a/cluster/services/patroni/haproxy.nix b/cluster/services/patroni/haproxy.nix new file mode 100644 index 0000000..6a4e7b8 --- /dev/null +++ b/cluster/services/patroni/haproxy.nix @@ -0,0 +1,41 @@ +{ cluster, ... }: + +let + inherit (cluster.config) vars; + + internalPort = cluster.config.links.patroni-pg-internal.portStr; + + checkPort = cluster.config.links.patroni-api.portStr; + + nodes = cluster.config.services.patroni.nodes.worker; + + getMeshIp = name: vars.mesh.${name}.meshIp; + + mkServerString = name: "server pg_ha_${name}_${internalPort} ${getMeshIp name}:${internalPort} maxconn 200 check port ${checkPort}"; +in + +{ + services.haproxy = { + enable = true; + config = '' + global + maxconn 200 + + defaults + log global + mode tcp + retries 2 + timeout client 30m + timeout connect 4s + timeout server 30m + timeout check 5s + + listen patroni + bind ${cluster.config.links.patroni-pg-access.tuple} + option httpchk + http-check expect status 200 + default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions + ${builtins.concatStringsSep " \n" (map mkServerString nodes)} + ''; + }; +} diff --git a/cluster/services/patroni/passwords/replication.age b/cluster/services/patroni/passwords/replication.age new file mode 100644 index 0000000..431e3f4 Binary files /dev/null and b/cluster/services/patroni/passwords/replication.age differ diff --git a/cluster/services/patroni/passwords/rewind.age b/cluster/services/patroni/passwords/rewind.age new file mode 100644 index 0000000..d1a1722 Binary files /dev/null and b/cluster/services/patroni/passwords/rewind.age differ diff --git a/cluster/services/patroni/passwords/superuser.age b/cluster/services/patroni/passwords/superuser.age new file mode 100644 index 0000000..63a8259 Binary files /dev/null and b/cluster/services/patroni/passwords/superuser.age differ diff --git a/cluster/services/patroni/worker.nix b/cluster/services/patroni/worker.nix new file mode 100644 index 0000000..f291365 --- /dev/null +++ b/cluster/services/patroni/worker.nix @@ -0,0 +1,76 @@ +{ aspect, cluster, config, lib, pkgs, ... }: + +let + inherit (cluster.config) vars; + + getMeshIp = name: vars.mesh.${name}.meshIp; + + net = vars.meshNet.cidr; + + pg = pkgs.postgresql_14; + + baseDir = "/srv/storage/database/postgres-ha"; + + cfg = config.services.patroni; +in + +{ + imports = [ + aspect.modules.patroni + ]; + + age.secrets = lib.mapAttrs (_: file: { + inherit file; + mode = "0400"; + owner = "patroni"; + group = "patroni"; + }) vars.patroni.passwords; + + systemd.tmpfiles.rules = [ "d '${baseDir}' 0700 patroni patroni - -" ]; + services.patroni = { + enable = true; + name = vars.hostName; + postgresqlPackage = pg; + postgresqlDataDir ="${baseDir}/${pg.psqlSchema}"; + postgresqlPort = cluster.config.links.patroni-pg-internal.port; + restApiPort = cluster.config.links.patroni-api.port; + scope = "poseidon"; + namespace = "/patroni"; + + nodeIp = getMeshIp vars.hostName; + otherNodesIps = map getMeshIp cluster.config.services.patroni.otherNodes.worker; + raft = false; + softwareWatchdog = true; + settings = { + etcd3.hosts = map (x: x.tuple) (lib.attrValues vars.patroni.etcdNodesClient); + bootstrap.dcs = { + ttl = 30; + loop_wait = 10; + retry_timeout = 10; + maximum_lag_on_failover = 1024 * 1024; + }; + postgresql = { + use_pg_rewind = true; + use_slots = true; + authentication = { + replication.username = "patronirep"; + rewind.username = "patronirew"; + superuser.username = "postgres"; + }; + parameters = { + listen_addresses = getMeshIp vars.hostName; + wal_level = "replica"; + hot_standby_feedback = "on"; + unix_socket_directories = "/tmp"; + }; + pg_hba = [ + "host replication patronirep ${net} scram-sha-256" + "host all patronirew ${net} scram-sha-256" + "host all postgres ${net} scram-sha-256" + "host all all 127.0.0.1/32 scram-sha-256" + ]; + }; + }; + environmentFiles = lib.mapAttrs (n: _: config.age.secrets.${n}.path) vars.patroni.passwords; + }; +} diff --git a/secrets.nix b/secrets.nix index 102d9ea..c805655 100644 --- a/secrets.nix +++ b/secrets.nix @@ -4,6 +4,9 @@ let systemKeys = x: x.ssh.id.publicKey or null; in with hosts; { + "cluster/services/patroni/passwords/replication.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ]; + "cluster/services/patroni/passwords/rewind.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ]; + "cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ]; "cluster/services/wireguard/mesh-keys/VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "cluster/services/wireguard/mesh-keys/prophet.age".publicKeys = max ++ map systemKeys [ prophet ]; "secrets/acme-dns-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];