Merge pull request #86 from privatevoid-net/svc-garage

Garage Service

cluster/services/storage/default.nix

@@ -1,12 +1,38 @@
+{ config, lib, ... }:
+
+let
+  meshIpForNode = name: config.vars.mesh.${name}.meshIp;
+in
+
 {
   services.storage = {
     nodes = {
       external = [ "prophet" ];
       heresy = [ "VEGAS" ];
+      garage = [ "checkmate" "prophet" "VEGAS" ];
+      garageInternal = [ "VEGAS" ];
+      garageExternal = [ "checkmate" "prophet" ];
     };
     nixos = {
       external = [ ./external.nix ];
       heresy = [ ./heresy.nix ];
+      garage = [
+        ./garage.nix
+        ./garage-options.nix
+        ./garage-layout.nix
+      ];
+      garageInternal = [ ./garage-internal.nix ];
+      garageExternal = [ ./garage-external.nix ];
     };
   };
+
+  hostLinks = lib.genAttrs config.services.storage.nodes.garage (name: {
+    garageRpc = {
+      ipv4 = meshIpForNode name;
+    };
+    garageS3 = {
+      protocol = "http";
+      ipv4 = meshIpForNode name;
+    };
+  });
 }

cluster/services/storage/garage-external.nix

@@ -0,0 +1,15 @@
+{ config, ... }:
+
+{
+  services.external-storage = {
+    underlays.garage = {
+      subUser = "sub1";
+      credentialsFile = ./secrets/storage-box-credentials.age;
+      path = "/garage/${config.networking.hostName}";
+      inherit (config.users.users.garage) uid;
+      inherit (config.users.groups.garage) gid;
+    };
+  };
+
+  services.garage.settings.data_dir = config.services.external-storage.underlays.garage.mountpoint;
+}

cluster/services/storage/garage-internal.nix

@@ -0,0 +1,11 @@
+let
+  dataDir = "/srv/storage/private/garage";
+in
+
+{
+  systemd.tmpfiles.rules = [
+    "d '${dataDir}' 0700 garage garage -"
+  ];
+
+  services.garage.settings.data_dir = dataDir;
+}

cluster/services/storage/garage-layout.nix

@@ -0,0 +1,14 @@
+{
+  system.ascensions.garage-layout = {
+    distributed = true;
+    requiredBy = [ "garage.service" ];
+    after = [ "garage.service" "garage-layout-init.service" ];
+    incantations = i: [ ];
+  };
+
+  services.garage.layout.initial = {
+    checkmate = { zone = "eu-central"; capacity = 1000; };
+    prophet = { zone = "eu-central"; capacity = 1000; };
+    VEGAS = { zone = "eu-central"; capacity = 1000; };
+  };
+}

cluster/services/storage/garage-options.nix

@@ -0,0 +1,232 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.garage;
+
+  garageShellLibrary = pkgs.writeText "garage-shell-library.sh" ''
+    getNodeId() {
+      nodeId=""
+      while [[ -z "$nodeId" ]]; do
+        nodeId="$(garage status | grep 'NO ROLE ASSIGNED' | grep -wm1 "$1" | cut -d' ' -f1)"
+        if [[ $? -ne 0 ]]; then
+          echo "Waiting for node $1 to appear..." 2>/dev/null
+          sleep 1
+        fi
+      done
+      echo "$nodeId"
+    }
+    waitForGarage() {
+      while ! garage status >/dev/null 2>/dev/null; do
+        sleep 1
+      done
+    }
+    waitForGarageOperational() {
+      waitForGarage
+      while garage layout show | grep -qwm1 '^Current cluster layout version: 0'; do
+        sleep 1
+      done
+    }
+    # FIXME: returns bogus empty string when one of the lists is empty
+    diffAdded() {
+      comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
+    }
+    diffRemoved() {
+      comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
+    }
+    # FIXME: this does not handle list items with spaces
+    listKeys() {
+      garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
+    }
+    ensureKeys() {
+      old="$(listKeys)"
+      if [[ -z "$1" ]]; then
+        for key in $old; do
+          garage key delete --yes "$key"
+        done
+      elif [[ -z "$old" ]]; then
+        for key in $1; do
+          # don't print secret key
+          garage key new --name "$key" >/dev/null
+          echo Key "$key" was created.
+        done
+      else
+        diffAdded "$old" "$1" | while read key; do
+          # don't print secret key
+          garage key new --name "$key" >/dev/null
+          echo Key "$key" was created.
+        done
+        diffRemoved "$old" "$1" | while read key; do
+          garage key delete --yes "$key"
+        done
+      fi
+    }
+    listBuckets() {
+      garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
+    }
+    ensureBuckets() {
+      old="$(listBuckets)"
+      if [[ -z "$1" ]]; then
+        for bucket in $old; do
+          garage bucket delete --yes "$bucket"
+        done
+      elif [[ -z "$old" ]]; then
+        for bucket in $1; do
+          garage bucket create "$bucket"
+        done
+      else
+        diffAdded "$old" "$1" | while read bucket; do
+          garage bucket create "$bucket"
+        done
+        diffRemoved "$old" "$1" | while read bucket; do
+          garage bucket delete --yes "$bucket"
+        done
+      fi
+    }
+  '';
+in
+
+{
+  options.services.garage = with lib; {
+    layout.initial = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule {
+        options = {
+          zone = mkOption {
+            type = types.str;
+          };
+          capacity = mkOption {
+            type = types.ints.positive;
+          };
+        };
+      });
+    };
+    keys = mkOption {
+      type = with types; attrsOf (submodule {
+        options.allow = {
+          createBucket = mkOption {
+            description = "Allow the key to create new buckets.";
+            type = bool;
+            default = false;
+          };
+        };
+      });
+      default = {};
+    };
+    buckets = mkOption {
+      type = with types; attrsOf (submodule {
+        options = {
+          allow = mkOption {
+            description = "List of permissions to grant on this bucket, grouped by key name.";
+            type = attrsOf (listOf (enum [ "read" "write" "owner" ]));
+            default = {};
+          };
+          quotas = {
+            maxObjects = mkOption {
+              description = "Maximum number of objects in this bucket. Null for unlimited.";
+              type = nullOr ints.positive;
+              default = null;
+            };
+            maxSize = mkOption {
+              description = "Maximum size of this bucket in bytes. Null for unlimited.";
+              type = nullOr ints.positive;
+              default = null;
+            };
+          };
+        };
+      });
+      default = {};
+    };
+  };
+
+  config = {
+    system.extraIncantations = {
+      runGarage = i: script: i.execShellWith [ config.services.garage.package ] ''
+        source ${garageShellLibrary}
+        waitForGarage
+        ${script}
+      '';
+    };
+
+    systemd.services = {
+      garage-layout-init = {
+        distributed.enable = true;
+        wantedBy = [ "garage.service" "multi-user.target" ];
+        wants = [ "garage.service" ];
+        after = [ "garage.service" ];
+        path = [ config.services.garage.package ];
+
+        serviceConfig = {
+          Type = "oneshot";
+          TimeoutStartSec = "1800s";
+          Restart = "on-failure";
+          RestartSec = "10s";
+        };
+        script = ''
+          source ${garageShellLibrary}
+          waitForGarage
+
+          if [[ "$(garage layout show | grep -m1 '^Current cluster layout version:' | cut -d: -f2 | tr -d ' ')" != "0" ]]; then
+            exit 0
+          fi
+
+          ${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: layout: ''
+            garage layout assign -z '${layout.zone}' -c '${toString layout.capacity}' "$(getNodeId '${name}')"
+          '') cfg.layout.initial)}
+
+          garage layout apply --version 1
+        '';
+      };
+      garage-apply = {
+        distributed.enable = true;
+        wantedBy = [ "garage.service" "multi-user.target" ];
+        wants = [ "garage.service" ];
+        after = [ "garage.service" "garage-layout-init.service" ];
+        path = [ config.services.garage.package ];
+
+        serviceConfig = {
+          Type = "oneshot";
+          TimeoutStartSec = "1800s";
+          Restart = "on-failure";
+          RestartSec = "10s";
+        };
+        script = ''
+          source ${garageShellLibrary}
+          waitForGarageOperational
+
+          ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
+          ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
+
+          # key permissions
+          ${lib.pipe cfg.keys [
+            (lib.mapAttrsToList (key: kCfg: ''
+              garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
+            ''))
+            (lib.concatStringsSep "\n")
+          ]}
+
+          # bucket permissions
+          ${lib.pipe cfg.buckets [
+            (lib.mapAttrsToList (bucket: bCfg:
+              lib.mapAttrsToList (key: perms: ''
+                garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
+                garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
+              '') bCfg.allow
+            ))
+            lib.flatten
+            (lib.concatStringsSep "\n")
+          ]}
+
+          # bucket quotas
+          ${lib.pipe cfg.buckets [
+            (lib.mapAttrsToList (bucket: bCfg: ''
+              garage bucket set-quotas '${bucket}' \
+                --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
+                --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
+            ''))
+            (lib.concatStringsSep "\n")
+          ]}
+        '';
+      };
+    };
+  };
+}

cluster/services/storage/garage.nix

@@ -0,0 +1,71 @@
+{ cluster, config, depot, lib, ... }:
+
+let
+  inherit (cluster.config) vars;
+  inherit (config.networking) hostName;
+
+  links = cluster.config.hostLinks.${hostName};
+
+  cfg = config.services.garage;
+
+  # fixed uid so we know how to mount the underlay
+  uid = 777;
+  gid = 777;
+in
+
+{
+  age.secrets.garageRpcSecret = {
+    file = ./secrets/garage-rpc-secret.age;
+    owner = "garage";
+    group = "garage";
+  };
+
+  services.garage = {
+    enable = true;
+    package = depot.packages.garage;
+    settings = {
+      replication_mode = 3;
+      block_size = 16 * 1024 * 1024;
+      db_engine = "lmdb";
+      metadata_dir = "/var/lib/garage-metadata";
+      rpc_bind_addr = links.garageRpc.tuple;
+      rpc_public_addr = links.garageRpc.tuple;
+      rpc_secret_file = config.age.secrets.garageRpcSecret.path;
+      consul_discovery = {
+        consul_http_addr = "http://127.0.0.1:8500";
+        api = "agent";
+        service_name = "garage-discovery";
+      };
+      s3_api = {
+        api_bind_addr = links.garageS3.tuple;
+        s3_region = "us-east-1";
+      };
+    };
+  };
+
+  users = {
+    users.garage = {
+      inherit uid;
+      group = "garage";
+    };
+    groups.garage = {
+      inherit gid;
+    };
+  };
+
+  systemd.services.garage = {
+    unitConfig = {
+      RequiresMountsFor = [ cfg.settings.data_dir ];
+    };
+    serviceConfig = {
+      IPAddressDeny = [ "any" ];
+      IPAddressAllow = [ "127.0.0.1/8" vars.meshNet.cidr ];
+      DynamicUser = false;
+      PrivateTmp = true;
+      ProtectSystem = true;
+      User = "garage";
+      Group = "garage";
+      StateDirectory = lib.removePrefix "/var/lib/" cfg.settings.metadata_dir;
+    };
+  };
+}

cluster/services/storage/secrets/garage-rpc-secret.age

@@ -0,0 +1,17 @@
+age-encryption.org/v1
+-> ssh-ed25519 NO562A WymFH+09McLeT2+o487RiKzExeG7QKTwwuERimfVygs
+dLTYSgo35f3cvhRWzhnutjSSRChBhhdX+87Yd6H6A5w
+-> ssh-ed25519 5/zT0w izNXf7mqdiDyc6AN6kbQfMHEtGxiI6+DnyiVxaSqFDo
+mXu7h0oY4UgiMojcNKedWXwvnQJmq4wyFkE1ZVTlXSk
+-> ssh-ed25519 TCgorQ 7ueU0wcT7A5bI6/JVKuIAj/jhh7TuVEfDCullS/Ws0c
+nri4g0jk8hdGMYgxjarklPBmzEBfXv2tgys8xPJ0UVI
+-> ssh-ed25519 d3WGuA 3LNotYdeRf4QP7bEdjx9NHInFVgRCFq91xjixu96mBI
+Gj0LYo6dTVIcSOXTtqdug+zX0l6UIigBu5obEgIMK00
+-> ssh-ed25519 YIaSKQ DkCXHGHVP92yEBKlELG/mWPkbq4lXoAmFVXmemC6Sxw
+YE4kSaNFVcS3txw94o5WTOKXe6xYbQALyzxBWDlZsCA
+-> ;",X-grease &l-# ,k
+FUMCq19iC2wN9U5NBYrHmCdUKB+p5AOnYGiA3LeWTJK3f899Bht0ZbSl7kL1l4w8
+4+n1ZbUITvPNOg
+--- Fri66T6QtwEkou+ThWbOy0m651htF4yedOH6tL0VX0E
+…Ѹ·$ýª€ð\÷»»½Ï%¶eè¸].IÍí»+G<>Ó˜ É
+A–/
kÀÃ8¿¯ÇiT“¿#•ê{é4µŽò@¿èh8¾I	\¹Mgµì·Kö¾ÕGx×è°aÿð§qO

cluster/services/storage/secrets/storage-box-credentials.age

@@ -1,13 +1,17 @@
 age-encryption.org/v1
--> ssh-ed25519 NO562A zJvkOwBBlON92XKax+VTvXyqGFajgwzt091jNrRzjjI
+-> ssh-ed25519 NO562A wZnh+3ROEclIIvMM4EhbUqytTwEVnODn9ayWAw81QGI
-1a8/gXx4DQu03Rewebxdz5BpEi3zI7DXTOcY2OLIUck
+id77DlK5BwGcJI79icF1dbfODRpAzuW/lmnNFQ9f8lQ
--> ssh-ed25519 5/zT0w cR4KCvOxvEV5KhEL+4HEcwKv5dKC4dT+yVok8P6Y/1c
+-> ssh-ed25519 5/zT0w +HimE5fwHyw00Mrl+A0OXw3unuqrRL7xBXsn3kLRFy8
-UCPh+WJ9opCFkDZgGBzyep8xZ58P6jLP1thMOpsx5rQ
+PoVKwvqyWGlPBaQ1ZTGd9gK1kc5w18z7hPJp1GAbZ1E
--> ssh-ed25519 d3WGuA nwgv+GeEw1Yc6gYCN39zfDyFg/NGxazU9tMkykmwhBc
+-> ssh-ed25519 TCgorQ oFe93M5WY5ovlHaNLBwA8LMRcydHtIt66IRzCmnxyQ4
-b+1EyRRSCvUWdMKcsEZA5BAnqQHO703Bwp5xtLPYok4
+ni+pTTEFop45tAEBUz6zne9Xgi42+gMTdoVnAQoZUls
--> ssh-ed25519 YIaSKQ FtSyO68mVkMEbMZV9w/H4tj7ms7gKOT+jB9n5MG5NAE
+-> ssh-ed25519 d3WGuA g3Ku++27IcH9g3xa5NqXz1itzMkIg+qoVo+yX25K1Wk
-B8Id7KRU9a19oWfmDcCN4zUb04AMZ4Z+AETyCvxeC7c
+Li5Ni2LbyFL8Bv/yCY5pwvK3/y5bS/quMvxnlwu2/7g
--> zAL;6ee-grease O]^kl 0`0G tf+sVH= `+K,h=HL
+-> ssh-ed25519 YIaSKQ 2SMv68txiKxrx+fs2+zMMCEa2SP4appPHlBZgPRRDH8
-ZO4H8I1upQszxI6GBalw1hUhSAhRRhhePbE
+xUp17uDntP1VUqrKMh0Hj73TcJh2o2LT7jaR4q7PVjY
---- lfbOCcNCO7rc1os2qEc9Bj0s6+bxJujVzmdL3lWdm3g
+-> e['-grease
-‡ÎeýÒm1ùuUVÈÏò#ÍjTóq–‘e=2Ìo¢ò9y2«3 õ•á?ù©jT1©‹¤œüGBAôïÇ_ü¹À3øÂg1-rL)Ë–gR	<09>
+ZqwKkCj0096w+J1bHZS2kubJ3egBdMrxFVE12g7AMsKSmq6bC1HyWsFJ2ZMNNVX3
+L04
+--- C5zj/EiQIzWfVY4XJp0eNPTfLFXud+cMOBR9/43e1jA
+³õG|HhG…ˆ“r–˜VËɲX9÷UÕ<55>
+Ýz˜"¥»Zƒ-“{KzÞ<>µM‡j%*h^V¶EŽÌÒZ¶\¡V>óuYÖ£&ËýD„BŠ)

modules/ascensions/default.nix

@@ -85,6 +85,7 @@ in
           targetLevel = toString (length incantations);
         in {
           description = "Ascension for ${name}";
+          wantedBy = [ "multi-user.target" ];
           inherit (asc) requiredBy before;
           after = asc.after ++ (lib.optional asc.distributed "consul.service");
           serviceConfig.Type = "oneshot";

modules/consul-distributed-services/default.nix

@@ -4,6 +4,9 @@ with lib;
 
 let
   consul = config.services.consul.package;
+
+  consulCfg = config.services.consul.extraConfig;
+  consulHttpAddr = "${consulCfg.addresses.http or "127.0.0.1"}:${toString (consulCfg.ports.http or 8500)}";
 in
 {
   options.systemd.services = mkOption {
@@ -31,11 +34,19 @@ in
         ''${@}
       '';
 
+      waitForConsul = pkgs.writeShellScript "wait-for-consul" ''
+        while ! ${consul}/bin/consul lock --name="pre-flight-check" --n=${toString cfg.replicas} --shell=false "$1" ${pkgs.coreutils}/bin/true; do
+          sleep 1
+        done
+      '';
+
       hasSpecialPrefix = elem (substring 0 1 ExecStart) [ "@" "-" ":" "+" "!" ];
     in assert !hasSpecialPrefix; pkgs.writeTextDir "etc/systemd/system/${n}.service.d/distributed.conf" ''
       [Service]
+      ExecStartPre=${waitForConsul} 'services/${n}%i'
       ExecStart=
       ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
+      Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
       ${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
       ${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
     ''))

modules/consul-service-registry/default.nix

@@ -7,13 +7,18 @@ let
 
   consul = "${config.services.consul.package}/bin/consul";
 
+  consulCfg = config.services.consul.extraConfig;
+  consulHttpAddr = "${consulCfg.addresses.http or "127.0.0.1"}:${toString (consulCfg.ports.http or 8500)}";
+
   consulRegisterScript = pkgs.writeShellScript "consul-register" ''
+    export CONSUL_HTTP_ADDR='${consulHttpAddr}'
     while ! ${consul} services register "$1"; do
       sleep 1
     done
   '';
 
   consulDeregisterScript = pkgs.writeShellScript "consul-deregister" ''
+    export CONSUL_HTTP_ADDR='${consulHttpAddr}'
     for i in {1..5}; do
       if ${consul} services deregister "$1"; then
         break

modules/external-storage/default.nix

@@ -49,6 +49,10 @@ in
           "credentials=${config.age.secrets."cifsCredentials-${name}".path}"
           "dir_mode=0700"
           "file_mode=0600"
+          "uid=${toString ul.uid}"
+          "gid=${toString ul.gid}"
+          "forceuid"
+          "forcegid"
           "seal"
           "hard"
           "resilienthandles"

modules/external-storage/underlay-type.nix

@@ -28,5 +28,13 @@ with lib;
       type = types.path;
       default = "/";
     };
+    uid = mkOption {
+      type = types.int;
+      default = 0;
+    };
+    gid = mkOption {
+      type = types.int;
+      default = 0;
+    };
   };
 }

packages/checks/default.nix

@@ -7,6 +7,12 @@
         inherit (self) nixosModules;
       };
 
+      garage = pkgs.callPackage ./garage.nix {
+        inherit (self'.packages) garage;
+        inherit (self) nixosModules;
+        inherit (config) cluster;
+      };
+
       jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix {
         inherit (self'.packages) jellyfin;
         inherit (config) cluster;

packages/checks/garage.nix

@@ -0,0 +1,141 @@
+{ testers, nixosModules, cluster, garage }:
+
+testers.runNixOSTest {
+  name = "garage";
+
+  imports = [
+    ./modules/consul.nix
+  ];
+
+  nodes = let
+    common = { config, lib, ... }: let
+      inherit (config.networking) hostName primaryIPAddress;
+    in {
+      imports = lib.flatten [
+        ./modules/nixos/age-dummy-secrets.nix
+        nixosModules.ascensions
+        nixosModules.systemd-extras
+        nixosModules.consul-distributed-services
+        cluster.config.services.storage.nixos.garage
+        cluster.config.services.storage.nixos.garageInternal
+      ];
+      config = {
+        _module.args = {
+          depot.packages = { inherit garage; };
+          cluster.config = {
+            hostLinks.${hostName} = {
+              garageRpc.tuple = "${primaryIPAddress}:3901";
+              garageS3.tuple = "${primaryIPAddress}:8080";
+            };
+            vars.meshNet.cidr = "192.168.0.0/16";
+          };
+        };
+        environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+        networking.firewall.allowedTCPPorts = [ 3901 8080 ];
+        services.garage = {
+          settings.consul_discovery.consul_http_addr = lib.mkForce "http://consul:8500";
+          layout.initial = lib.mkOverride 51 {
+            garage1 = { zone = "dc1"; capacity = 1000; };
+            garage2 = { zone = "dc1"; capacity = 1000; };
+            garage3 = { zone = "dc1"; capacity = 1000; };
+          };
+        };
+        system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
+        specialisation.modifiedLayout = {
+          inheritParentConfig = true;
+          configuration = {
+            services.garage = {
+              layout.initial = lib.mkForce {
+                garage1 = { zone = "dc1"; capacity = 2000; };
+                garage2 = { zone = "dc1"; capacity = 1000; };
+                garage3 = { zone = "dc1"; capacity = 1000; };
+              };
+              keys.testKey.allow.createBucket = true;
+              buckets = {
+                bucket1 = {
+                  allow.testKey = [ "read" "write" ];
+                  quotas = {
+                    maxObjects = 300;
+                    maxSize = 400 * 1024 * 1024;
+                  };
+                };
+                bucket2 = {
+                  allow.testKey = [ "read" ];
+                };
+              };
+            };
+            system.ascensions.garage-layout.incantations = lib.mkForce (i: [
+              (i.runGarage ''
+                garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
+                garage layout apply --version 2
+              '')
+            ]);
+          };
+        };
+      };
+    };
+  in {
+    garage1.imports = [ common ];
+    garage2.imports = [ common ];
+    garage3.imports = [ common ];
+  };
+
+  testScript = { nodes, ... }: /*python*/ ''
+    nodes = [garage1, garage2, garage3]
+
+    start_all()
+
+    with subtest("should bootstrap new cluster"):
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+    with subtest("should apply new layout with ascension"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl is-active ascend-garage-layout)" != activating')
+
+      for node in nodes:
+        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should apply new layout from scratch"):
+      for node in nodes:
+        node.systemctl("stop garage.service")
+        node.succeed("rm -rf /var/lib/garage-metadata")
+
+      for node in nodes:
+        node.systemctl("start garage.service")
+
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should create specified buckets and keys"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
+      garage1.succeed("garage key list | grep testKey")
+      garage1.succeed("garage bucket list | grep bucket1")
+      garage1.succeed("garage bucket list | grep bucket2")
+
+    with subtest("should delete unspecified buckets and keys"):
+      garage1.succeed("garage bucket create unwantedbucket")
+      garage1.succeed("garage key new --name unwantedkey")
+      garage1.succeed("systemctl restart garage-apply.service")
+
+      garage1.fail("garage key list | grep unwantedkey")
+      garage1.fail("garage bucket list | grep unwantedbucket")
+  '';
+}

packages/checks/modules/nixos/age-dummy-secrets.nix

@@ -0,0 +1,33 @@
+{ config, lib, ... }:
+with lib;
+
+let
+  t = {
+    string = default: mkOption {
+      type = types.str;
+      inherit default;
+    };
+  };
+in
+
+{
+  options.age.secrets = mkOption {
+    type = types.attrsOf (types.submodule ({ name, config, ... }: {
+      options = {
+        file = mkSinkUndeclaredOptions {};
+        owner = t.string "root";
+        group = t.string "root";
+        mode = t.string "400";
+        path = t.string "/etc/dummy-secrets/${name}";
+      };
+    }));
+  };
+  config.environment.etc = mapAttrs' (name: secret: {
+    name = removePrefix "/etc/" secret.path;
+    value = mapAttrs (const mkDefault) {
+      user = secret.owner;
+      inherit (secret) mode group;
+      text = builtins.hashString "md5" name;
+    };
+  }) config.age.secrets;
+}

packages/patched-derivations.nix

@@ -53,6 +53,8 @@ super: rec {
 
   forgejo = patch super.forgejo "patches/base/forgejo";
 
+  garage = patch super.garage_0_8 "patches/base/garage";
+
   jellyfin = patch (super.jellyfin.override {
     ffmpeg = super.ffmpeg.override {
       withMfx = true;

patches/base/garage/print-chill-pills.patch

@@ -0,0 +1,264 @@
+diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs
+index bbde204..7bed4b0 100644
+--- a/src/db/bin/convert.rs
++++ b/src/db/bin/convert.rs
+@@ -4,6 +4,28 @@ use garage_db::*;
+ 
+ use clap::Parser;
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
++macro_rules! print {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    write!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ /// K2V command line interface
+ #[derive(Parser, Debug)]
+ #[clap(author, version, about, long_about = None)]
+diff --git a/src/db/lib.rs b/src/db/lib.rs
+index 11cae4e..ffef3fa 100644
+--- a/src/db/lib.rs
++++ b/src/db/lib.rs
+@@ -25,6 +25,18 @@ use std::sync::Arc;
+ 
+ use err_derive::Error;
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ #[derive(Clone)]
+ pub struct Db(pub(crate) Arc<dyn IDb>);
+ 
+diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs
+index 0d73588..6bf4ecc 100644
+--- a/src/garage/cli/cmd.rs
++++ b/src/garage/cli/cmd.rs
+@@ -13,6 +13,28 @@ use garage_model::helper::error::Error as HelperError;
+ use crate::admin::*;
+ use crate::cli::*;
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
++macro_rules! print {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    write!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ pub async fn cli_command_dispatch(
+ 	cmd: Command,
+ 	system_rpc_endpoint: &Endpoint<SystemRpc, ()>,
+diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs
+index 20813f1..f4baea2 100644
+--- a/src/garage/cli/init.rs
++++ b/src/garage/cli/init.rs
+@@ -2,6 +2,18 @@ use std::path::PathBuf;
+ 
+ use garage_util::error::*;
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ pub const READ_KEY_ERROR: &str = "Unable to read node key. It will be generated by your garage node the first time is it launched. Ensure that your garage node is currently running. (The node key is supposed to be stored in your metadata directory.)";
+ 
+ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
+diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs
+index 3884bb9..ef55a66 100644
+--- a/src/garage/cli/layout.rs
++++ b/src/garage/cli/layout.rs
+@@ -8,6 +8,28 @@ use garage_rpc::*;
+ 
+ use crate::cli::*;
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
++macro_rules! print {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    write!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ pub async fn cli_layout_command_dispatch(
+ 	cmd: LayoutOperation,
+ 	system_rpc_endpoint: &Endpoint<SystemRpc, ()>,
+diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs
+index 2c6be2f..db6f25d 100644
+--- a/src/garage/cli/util.rs
++++ b/src/garage/cli/util.rs
+@@ -17,6 +17,28 @@ use garage_model::s3::version_table::Version;
+ 
+ use crate::cli::structs::WorkerListOpt;
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
++macro_rules! print {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    write!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ pub fn print_bucket_list(bl: Vec<Bucket>) {
+ 	println!("List of buckets:");
+ 
+diff --git a/src/k2v-client/bin/k2v-cli.rs b/src/k2v-client/bin/k2v-cli.rs
+index cdd63cc..dfa4df4 100644
+--- a/src/k2v-client/bin/k2v-cli.rs
++++ b/src/k2v-client/bin/k2v-cli.rs
+@@ -11,6 +11,28 @@ use rusoto_core::Region;
+ 
+ use clap::{Parser, Subcommand};
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
++macro_rules! print {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    write!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ /// K2V command line interface
+ #[derive(Parser, Debug)]
+ #[clap(author, version, about, long_about = None)]
+diff --git a/src/rpc/layout.rs b/src/rpc/layout.rs
+index 1030e3a..47eca49 100644
+--- a/src/rpc/layout.rs
++++ b/src/rpc/layout.rs
+@@ -10,6 +10,28 @@ use garage_util::error::*;
+ 
+ use crate::ring::*;
+ 
++use std::io::Write;
++
++macro_rules! println {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    writeln!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
++macro_rules! print {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    write!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ /// The layout of the cluster, i.e. the list of roles
+ /// which are assigned to each cluster node
+ #[derive(Clone, Debug, Serialize, Deserialize)]
+diff --git a/src/util/formater.rs b/src/util/formater.rs
+index 2ea53eb..cc7d8a4 100644
+--- a/src/util/formater.rs
++++ b/src/util/formater.rs
+@@ -1,3 +1,15 @@
++use std::io::Write;
++
++macro_rules! print {
++  () => (print!("\n"));
++  ($fmt:expr) => ({
++    write!(std::io::stdout(), $fmt).unwrap_or(())
++  });
++  ($fmt:expr, $($arg:tt)*) => ({
++    write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
++  })
++}
++
+ pub fn format_table_to_string(data: Vec<String>) -> String {
+ 	let data = data
+ 		.iter()

secrets.nix

@@ -44,7 +44,8 @@ in with hosts;
   "cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
   "cluster/services/storage/secrets/heresy-encryption-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/storage/secrets/external-storage-encryption-key-prophet.age".publicKeys = max ++ map systemKeys [ prophet ];
-  "cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
+  "cluster/services/storage/secrets/garage-rpc-secret.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
+  "cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
   "cluster/services/wireguard/mesh-keys/checkmate.age".publicKeys = max ++ map systemKeys [ checkmate ];
   "cluster/services/wireguard/mesh-keys/thunderskin.age".publicKeys = max ++ map systemKeys [ thunderskin ];
   "cluster/services/wireguard/mesh-keys/VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ];