cluster/services/patroni: init

cluster/services/patroni/default.nix

@@ -0,0 +1,49 @@
+{ config, lib, ... }:
+
+let
+  inherit (config.vars) hosts;
+
+  cfg = config.services.patroni;
+
+  renameToLink = mode: n: v: lib.nameValuePair "patroni-etcd-node-${mode}-${n}" v;
+
+  genLinks = mode: nodes: f: lib.mapAttrs' (renameToLink mode) (lib.genAttrs nodes f);
+
+  getMeshIp = name: config.vars.mesh.${name}.meshIp;
+
+  mkLink = name: {
+    ipv4 = getMeshIp name;
+    protocol = "http";
+  };
+in
+{
+  vars.patroni = {
+    etcdNodes = lib.genAttrs cfg.nodes.etcd (name: config.links."patroni-etcd-node-peer-${name}");
+    etcdNodesClient = lib.genAttrs cfg.nodes.etcd (name: config.links."patroni-etcd-node-client-${name}");
+    etcdExtraNodes = [ "fly=http://10.1.1.151:2380" ];
+    passwords = {
+      PATRONI_REPLICATION_PASSWORD = ./passwords/replication.age;
+      PATRONI_SUPERUSER_PASSWORD = ./passwords/superuser.age;
+      PATRONI_REWIND_PASSWORD = ./passwords/rewind.age;
+    };
+  };
+  links = genLinks "client" cfg.nodes.etcd mkLink
+    // genLinks "peer" cfg.nodes.etcd mkLink
+    // {
+    patroni-pg-internal.ipv4 = "0.0.0.0";
+    patroni-api.ipv4 = "0.0.0.0";
+    patroni-pg-access.ipv4 = "127.0.0.1";
+  };
+  services.patroni = {
+    nodes = {
+      worker = [ "VEGAS" "prophet" ];
+      etcd = [ "VEGAS" "prophet" ];
+      haproxy = [ "VEGAS" "prophet" ];
+    };
+    nixos = {
+      worker = ./worker.nix;
+      etcd = ./etcd.nix;
+      haproxy = ./haproxy.nix;
+    };
+  };
+}

cluster/services/patroni/etcd.nix

@@ -0,0 +1,21 @@
+{ cluster, lib, pkgs, ... }:
+
+let
+  inherit (cluster.config) vars;
+
+  getEtcdUrl = name: vars.patroni.etcdNodes.${name}.url;
+
+  mkMember = n: "${n}=${getEtcdUrl n}";
+in
+
+{
+  services.etcd = {
+    enable = true;
+    dataDir = "/srv/storage/private/etcd";
+    initialCluster = (map mkMember cluster.config.services.patroni.nodes.etcd) ++ vars.patroni.etcdExtraNodes;
+    listenPeerUrls = lib.singleton vars.patroni.etcdNodes.${vars.hostName}.url;
+    listenClientUrls = lib.singleton vars.patroni.etcdNodesClient.${vars.hostName}.url;
+  };
+  # run on any architecture
+  systemd.services.etcd.environment.ETCD_UNSUPPORTED_ARCH = pkgs.go.GOARCH;
+}

cluster/services/patroni/haproxy.nix

@@ -0,0 +1,41 @@
+{ cluster, ... }:
+
+let
+  inherit (cluster.config) vars;
+
+  internalPort = cluster.config.links.patroni-pg-internal.portStr;
+
+  checkPort = cluster.config.links.patroni-api.portStr;
+
+  nodes = cluster.config.services.patroni.nodes.worker;
+
+  getMeshIp = name: vars.mesh.${name}.meshIp;
+
+  mkServerString = name: "server pg_ha_${name}_${internalPort} ${getMeshIp name}:${internalPort} maxconn 200 check port ${checkPort}";
+in
+
+{
+  services.haproxy = {
+    enable = true;
+    config = ''
+      global
+          maxconn 200
+      
+      defaults
+          log global
+          mode tcp
+          retries 2
+          timeout client 30m
+          timeout connect 4s
+          timeout server 30m
+          timeout check 5s
+      
+      listen patroni
+          bind ${cluster.config.links.patroni-pg-access.tuple}
+          option httpchk
+          http-check expect status 200
+          default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+          ${builtins.concatStringsSep "    \n" (map mkServerString nodes)}
+    '';
+  };
+}

cluster/services/patroni/worker.nix

@@ -0,0 +1,76 @@
+{ aspect, cluster, config, lib, pkgs, ... }:
+
+let
+  inherit (cluster.config) vars;
+
+  getMeshIp = name: vars.mesh.${name}.meshIp;
+
+  net = vars.meshNet.cidr;
+
+  pg = pkgs.postgresql_14;
+
+  baseDir = "/srv/storage/database/postgres-ha";
+
+  cfg = config.services.patroni;
+in
+
+{
+  imports = [
+    aspect.modules.patroni
+  ];
+
+  age.secrets = lib.mapAttrs (_: file: {
+    inherit file;
+    mode = "0400";
+    owner = "patroni";
+    group = "patroni";
+  }) vars.patroni.passwords;
+
+  systemd.tmpfiles.rules = [ "d '${baseDir}' 0700 patroni patroni - -" ];
+  services.patroni = {
+    enable = true;
+    name = vars.hostName;
+    postgresqlPackage = pg;
+    postgresqlDataDir ="${baseDir}/${pg.psqlSchema}";
+    postgresqlPort = cluster.config.links.patroni-pg-internal.port;
+    restApiPort = cluster.config.links.patroni-api.port;
+    scope = "poseidon";
+    namespace = "/patroni";
+
+    nodeIp = getMeshIp vars.hostName;
+    otherNodesIps = map getMeshIp cluster.config.services.patroni.otherNodes.worker;
+    raft = false;
+    softwareWatchdog = true;
+    settings = {
+      etcd3.hosts = map (x: x.tuple) (lib.attrValues vars.patroni.etcdNodesClient);
+      bootstrap.dcs = {
+        ttl = 30;
+        loop_wait = 10;
+        retry_timeout = 10;
+        maximum_lag_on_failover = 1024 * 1024;
+      };
+      postgresql = {
+        use_pg_rewind = true;
+        use_slots = true;
+        authentication = {
+          replication.username = "patronirep";
+          rewind.username = "patronirew";
+          superuser.username = "postgres";
+        };
+        parameters = {
+          listen_addresses = getMeshIp vars.hostName;
+          wal_level = "replica";
+          hot_standby_feedback = "on";
+          unix_socket_directories = "/tmp";
+        };
+        pg_hba = [
+          "host replication patronirep ${net} scram-sha-256"
+          "host all patronirew ${net} scram-sha-256"
+          "host all postgres ${net} scram-sha-256"
+          "host all all 127.0.0.1/32 scram-sha-256"
+        ];
+      };
+    };
+    environmentFiles = lib.mapAttrs (n: _: config.age.secrets.${n}.path) vars.patroni.passwords;
+  };
+}

secrets.nix

@@ -4,6 +4,9 @@ let
   systemKeys = x: x.ssh.id.publicKey or null;
 in with hosts;
 {
+  "cluster/services/patroni/passwords/replication.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
+  "cluster/services/patroni/passwords/rewind.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
+  "cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "cluster/services/wireguard/mesh-keys/VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/wireguard/mesh-keys/prophet.age".publicKeys = max ++ map systemKeys [ prophet ];
   "secrets/acme-dns-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];