From 2617a1404295fe69c79e747893f263fee88bd3fa Mon Sep 17 00:00:00 2001
From: Max <max@privatevoid.net>
Date: Tue, 23 Jul 2024 02:47:13 +0200
Subject: [PATCH] cluster/services/storage: test in simulacrum

---
 cluster/services/storage/default.nix          |   7 ++
 cluster/services/storage/garage.nix           |   2 +
 .../simulacrum/snakeoil-rpc-secret.nix        |   3 +
 cluster/services/storage/test.nix             | 106 ++++++++++++++++++
 4 files changed, 118 insertions(+)
 create mode 100644 cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix
 create mode 100644 cluster/services/storage/test.nix

diff --git a/cluster/services/storage/default.nix b/cluster/services/storage/default.nix
index d69aeec..20fe388 100644
--- a/cluster/services/storage/default.nix
+++ b/cluster/services/storage/default.nix
@@ -35,6 +35,8 @@ in
         ./garage.nix
         ./garage-options.nix
         ./garage-layout.nix
+      ] ++ lib.optionals config.simulacrum [
+        ./simulacrum/snakeoil-rpc-secret.nix
       ];
       garageConfig = [
         ./garage-gateway.nix
@@ -48,6 +50,11 @@ in
       garageInternal = [ ./garage-internal.nix ];
       garageExternal = [ ./garage-external.nix ];
     };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" "consul" "locksmith" ];
+      settings = ./test.nix;
+    };
   };
 
   links = {
diff --git a/cluster/services/storage/garage.nix b/cluster/services/storage/garage.nix
index a319ba1..93d0bcf 100644
--- a/cluster/services/storage/garage.nix
+++ b/cluster/services/storage/garage.nix
@@ -63,6 +63,8 @@ in
   };
 
   systemd.services.garage = {
+    requires = [ "consul-ready.service" ];
+    after = [ "consul-ready.service" ];
     unitConfig = {
       RequiresMountsFor = [ cfg.settings.data_dir ];
     };
diff --git a/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix b/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix
new file mode 100644
index 0000000..501bf1b
--- /dev/null
+++ b/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix
@@ -0,0 +1,3 @@
+{
+  environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+}
diff --git a/cluster/services/storage/test.nix b/cluster/services/storage/test.nix
new file mode 100644
index 0000000..2bd6d5e
--- /dev/null
+++ b/cluster/services/storage/test.nix
@@ -0,0 +1,106 @@
+{ cluster, lib, ... }:
+
+let
+  inherit (cluster.config.services.storage) nodes;
+
+  firstGarageNode = lib.elemAt nodes.garage 0;
+in
+
+{
+  nodes = lib.genAttrs nodes.garage (node: {
+    services.garage = {
+      layout.initial = lib.genAttrs nodes.garage (_: {
+        capacity = lib.mkOverride 51 1000;
+      });
+    };
+    specialisation.modifiedLayout = {
+      inheritParentConfig = true;
+      configuration = {
+        services.garage = {
+          layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
+          keys.testKey.allow.createBucket = true;
+          buckets = {
+            bucket1 = {
+              allow.testKey = [ "read" "write" ];
+              quotas = {
+                maxObjects = 300;
+                maxSize = 400 * 1024 * 1024;
+              };
+            };
+            bucket2 = {
+              allow.testKey = [ "read" ];
+            };
+          };
+        };
+        system.ascensions.garage-layout.incantations = lib.mkForce (i: [
+          (i.runGarage ''
+            garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
+            garage layout apply --version 2
+          '')
+        ]);
+      };
+    };
+  });
+
+  testScript = ''
+    import json
+    nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
+    garage1 = nodes[0]
+
+    start_all()
+
+    with subtest("should bootstrap new cluster"):
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+    with subtest("should apply new layout with ascension"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
+
+      for node in nodes:
+        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should apply new layout from scratch"):
+      for node in nodes:
+        node.systemctl("stop garage.service")
+        node.succeed("rm -rf /var/lib/garage-metadata")
+
+      for node in nodes:
+        node.systemctl("start garage.service")
+
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should create specified buckets and keys"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
+      garage1.succeed("garage key list | grep testKey")
+      garage1.succeed("garage bucket list | grep bucket1")
+      garage1.succeed("garage bucket list | grep bucket2")
+
+    with subtest("should delete unspecified buckets and keys"):
+      garage1.succeed("garage bucket create unwantedbucket")
+      garage1.succeed("garage key new --name unwantedkey")
+      garage1.succeed("systemctl restart garage-apply.service")
+
+      garage1.fail("garage key list | grep unwantedkey")
+      garage1.fail("garage bucket list | grep unwantedbucket")
+  '';
+}
+