cluster/services/storage: add garage

cluster/services/storage/default.nix

@@ -1,12 +1,37 @@
+{ config, lib, ... }:
+
+let
+  meshIpForNode = name: config.vars.mesh.${name}.meshIp;
+in
+
 {
   services.storage = {
     nodes = {
       external = [ "prophet" ];
       heresy = [ "VEGAS" ];
+      garage = [ "checkmate" "prophet" "VEGAS" ];
+      garageInternal = [ "VEGAS" ];
+      garageExternal = [ "checkmate" "prophet" ];
     };
     nixos = {
       external = [ ./external.nix ];
       heresy = [ ./heresy.nix ];
+      garage = [
+        ./garage.nix
+        ./garage-layout.nix
+      ];
+      garageInternal = [ ./garage-internal.nix ];
+      garageExternal = [ ./garage-external.nix ];
     };
   };
+
+  hostLinks = lib.genAttrs config.services.storage.nodes.garage (name: {
+    garageRpc = {
+      ipv4 = meshIpForNode name;
+    };
+    garageS3 = {
+      protocol = "http";
+      ipv4 = meshIpForNode name;
+    };
+  });
 }

cluster/services/storage/garage-external.nix

@@ -0,0 +1,15 @@
+{ config, ... }:
+
+{
+  services.external-storage = {
+    underlays.garage = {
+      subUser = "sub1";
+      credentialsFile = ./secrets/storage-box-credentials.age;
+      path = "/garage/${config.networking.hostName}";
+      inherit (config.users.users.garage) uid;
+      inherit (config.users.groups.garage) gid;
+    };
+  };
+
+  services.garage.settings.data_dir = config.services.external-storage.underlays.garage.mountpoint;
+}

cluster/services/storage/garage-internal.nix

@@ -0,0 +1,11 @@
+let
+  dataDir = "/srv/storage/private/garage";
+in
+
+{
+  systemd.tmpfiles.rules = [
+    "d '${dataDir}' 0700 garage garage -"
+  ];
+
+  services.garage.settings.data_dir = dataDir;
+}

cluster/services/storage/garage-layout.nix

@@ -0,0 +1,39 @@
+{ config, ... }:
+
+let
+  initialLayout = ''
+    garage layout assign -z eu-central -c 1000 28071b8673ad14c2 # checkmate
+    garage layout assign -z eu-central -c 1000 124d6acad43e5f70 # prophet
+    garage layout assign -z eu-central -c 1000 e354a1a70adc45c9 # VEGAS
+  '';
+in
+
+{
+  system.ascensions.garage-layout = {
+    distributed = true;
+    requiredBy = [ "garage.service" ];
+    after = [ "garage.service" "garage-layout-init.service" ];
+    incantations = i: [ ];
+  };
+
+  systemd.services.garage-layout-init = {
+    distributed.enable = true;
+    wantedBy = [ "garage.service" ];
+    after = [ "garage.service" ];
+    path = [ config.services.garage.package ];
+
+    script = ''
+      while ! garage status >/dev/null 2>/dev/null; do
+        sleep 1
+      done
+
+      if [[ "$(garage layout show | grep -m1 '^Current cluster layout version:' | cut -d: -f2 | tr -d ' ')" != "0" ]]; then
+        exit 0
+      fi
+
+      ${initialLayout}
+
+      garage layout apply --version 1
+    '';
+  };
+}

cluster/services/storage/garage.nix

@@ -0,0 +1,69 @@
+{ cluster, config, depot, lib, ... }:
+
+let
+  inherit (cluster.config) vars;
+  inherit (config.networking) hostName;
+
+  links = cluster.config.hostLinks.${hostName};
+
+  cfg = config.services.garage;
+
+  # fixed uid so we know how to mount the underlay
+  uid = 777;
+  gid = 777;
+in
+
+{
+  age.secrets.garageRpcSecret = {
+    file = ./secrets/garage-rpc-secret.age;
+    owner = "garage";
+    group = "garage";
+  };
+
+  services.garage = {
+    enable = true;
+    package = depot.packages.garage;
+    settings = {
+      replication_mode = 3;
+      block_size = 16 * 1024 * 1024;
+      db_engine = "lmdb";
+      metadata_dir = "/var/lib/garage-metadata";
+      rpc_bind_addr = links.garageRpc.tuple;
+      rpc_public_addr = links.garageRpc.tuple;
+      rpc_secret_file = config.age.secrets.garageRpcSecret.path;
+      consul_discovery = {
+        consul_http_addr = "http://127.0.0.1:8500";
+        api = "agent";
+        service_name = "garage-discovery";
+      };
+      s3_api = {
+        api_bind_addr = links.garageS3.tuple;
+        s3_region = "us-east-1";
+      };
+    };
+  };
+
+  users = {
+    users.garage = {
+      inherit uid;
+      group = "garage";
+    };
+    groups.garage = {
+      inherit gid;
+    };
+  };
+
+  systemd.services.garage = {
+    serviceConfig = {
+      IPAddressDeny = [ "any" ];
+      IPAddressAllow = [ "127.0.0.1/8" vars.meshNet.cidr ];
+      RequiresMountsFor = [ cfg.settings.data_dir ];
+      DynamicUser = false;
+      PrivateTmp = true;
+      ProtectSystem = true;
+      User = "garage";
+      Group = "garage";
+      StateDirectory = lib.removePrefix "/var/lib/" cfg.settings.metadata_dir;
+    };
+  };
+}

cluster/services/storage/secrets/garage-rpc-secret.age

@@ -0,0 +1,17 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A WymFH+09McLeT2+o487RiKzExeG7QKTwwuERimfVygs
+dLTYSgo35f3cvhRWzhnutjSSRChBhhdX+87Yd6H6A5w
+-> ssh-ed25519 5/zT0w izNXf7mqdiDyc6AN6kbQfMHEtGxiI6+DnyiVxaSqFDo
+mXu7h0oY4UgiMojcNKedWXwvnQJmq4wyFkE1ZVTlXSk
+-> ssh-ed25519 TCgorQ 7ueU0wcT7A5bI6/JVKuIAj/jhh7TuVEfDCullS/Ws0c
+nri4g0jk8hdGMYgxjarklPBmzEBfXv2tgys8xPJ0UVI
+-> ssh-ed25519 d3WGuA 3LNotYdeRf4QP7bEdjx9NHInFVgRCFq91xjixu96mBI
+Gj0LYo6dTVIcSOXTtqdug+zX0l6UIigBu5obEgIMK00
+-> ssh-ed25519 YIaSKQ DkCXHGHVP92yEBKlELG/mWPkbq4lXoAmFVXmemC6Sxw
+YE4kSaNFVcS3txw94o5WTOKXe6xYbQALyzxBWDlZsCA
+-> ;",X-grease &l-# ,k
+FUMCq19iC2wN9U5NBYrHmCdUKB+p5AOnYGiA3LeWTJK3f899Bht0ZbSl7kL1l4w8
+4+n1ZbUITvPNOg
+--- Fri66T6QtwEkou+ThWbOy0m651htF4yedOH6tL0VX0E
+…Ѹ·$ýª€ð\÷»»½Ï%¶eè¸].IÍí»+G<>Ó˜ É
+A–/
kÀÃ8¿¯ÇiT“¿#•ê{é4µŽò@¿èh8¾I	\¹Mgµì·Kö¾ÕGx×è°aÿð§qO

cluster/services/storage/secrets/storage-box-credentials.age

@@ -1,13 +1,17 @@
 age-encryption.org/v1
--> ssh-ed25519 NO562A zJvkOwBBlON92XKax+VTvXyqGFajgwzt091jNrRzjjI
+-> ssh-ed25519 NO562A wZnh+3ROEclIIvMM4EhbUqytTwEVnODn9ayWAw81QGI
-1a8/gXx4DQu03Rewebxdz5BpEi3zI7DXTOcY2OLIUck
+id77DlK5BwGcJI79icF1dbfODRpAzuW/lmnNFQ9f8lQ
--> ssh-ed25519 5/zT0w cR4KCvOxvEV5KhEL+4HEcwKv5dKC4dT+yVok8P6Y/1c
+-> ssh-ed25519 5/zT0w +HimE5fwHyw00Mrl+A0OXw3unuqrRL7xBXsn3kLRFy8
-UCPh+WJ9opCFkDZgGBzyep8xZ58P6jLP1thMOpsx5rQ
+PoVKwvqyWGlPBaQ1ZTGd9gK1kc5w18z7hPJp1GAbZ1E
--> ssh-ed25519 d3WGuA nwgv+GeEw1Yc6gYCN39zfDyFg/NGxazU9tMkykmwhBc
+-> ssh-ed25519 TCgorQ oFe93M5WY5ovlHaNLBwA8LMRcydHtIt66IRzCmnxyQ4
-b+1EyRRSCvUWdMKcsEZA5BAnqQHO703Bwp5xtLPYok4
+ni+pTTEFop45tAEBUz6zne9Xgi42+gMTdoVnAQoZUls
--> ssh-ed25519 YIaSKQ FtSyO68mVkMEbMZV9w/H4tj7ms7gKOT+jB9n5MG5NAE
+-> ssh-ed25519 d3WGuA g3Ku++27IcH9g3xa5NqXz1itzMkIg+qoVo+yX25K1Wk
-B8Id7KRU9a19oWfmDcCN4zUb04AMZ4Z+AETyCvxeC7c
+Li5Ni2LbyFL8Bv/yCY5pwvK3/y5bS/quMvxnlwu2/7g
--> zAL;6ee-grease O]^kl 0`0G tf+sVH= `+K,h=HL
+-> ssh-ed25519 YIaSKQ 2SMv68txiKxrx+fs2+zMMCEa2SP4appPHlBZgPRRDH8
-ZO4H8I1upQszxI6GBalw1hUhSAhRRhhePbE
+xUp17uDntP1VUqrKMh0Hj73TcJh2o2LT7jaR4q7PVjY
---- lfbOCcNCO7rc1os2qEc9Bj0s6+bxJujVzmdL3lWdm3g
+-> e['-grease
-‡ÎeýÒm1ùuUVÈÏò#ÍjTóq–‘e=2Ìo¢ò9y2«3 õ•á?ù©jT1©‹¤œüGBAôïÇ_ü¹À3øÂg1-rL)Ë–gR	<09>
+ZqwKkCj0096w+J1bHZS2kubJ3egBdMrxFVE12g7AMsKSmq6bC1HyWsFJ2ZMNNVX3
+L04
+--- C5zj/EiQIzWfVY4XJp0eNPTfLFXud+cMOBR9/43e1jA
+³õG|HhG…ˆ“r–˜VËɲX9÷UÕ<55>
+Ýz˜"¥»Zƒ-“{KzÞ<>µM‡j%*h^V¶EŽÌÒZ¶\¡V>óuYÖ£&ËýD„BŠ)

secrets.nix

@@ -44,7 +44,8 @@ in with hosts;
   "cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/storage/secrets/heresy-encryption-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/storage/secrets/external-storage-encryption-key-prophet.age".publicKeys = max ++ map systemKeys [ prophet ];
-  "cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
+  "cluster/services/storage/secrets/garage-rpc-secret.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
+  "cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
   "cluster/services/wireguard/mesh-keys/checkmate.age".publicKeys = max ++ map systemKeys [ checkmate ];
   "cluster/services/wireguard/mesh-keys/thunderskin.age".publicKeys = max ++ map systemKeys [ thunderskin ];
   "cluster/services/wireguard/mesh-keys/VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ];