From 7287fcb5db9f1eb32f8eef689661a448d451b3dc Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 23 Jul 2024 02:47:13 +0200 Subject: [PATCH] cluster/services/storage: test in simulacrum --- cluster/services/storage/default.nix | 8 ++ cluster/services/storage/garage.nix | 2 + .../simulacrum/snakeoil-rpc-secret.nix | 3 + .../services/storage/simulacrum/test-data.nix | 8 ++ cluster/services/storage/simulacrum/test.nix | 105 ++++++++++++++++++ 5 files changed, 126 insertions(+) create mode 100644 cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix create mode 100644 cluster/services/storage/simulacrum/test-data.nix create mode 100644 cluster/services/storage/simulacrum/test.nix diff --git a/cluster/services/storage/default.nix b/cluster/services/storage/default.nix index e4da92c..4ab6ac0 100644 --- a/cluster/services/storage/default.nix +++ b/cluster/services/storage/default.nix @@ -8,6 +8,7 @@ in imports = [ ./options.nix ./incandescence.nix + ./simulacrum/test-data.nix ]; services.storage = { @@ -36,6 +37,8 @@ in ./garage.nix ./garage-options.nix ./garage-layout.nix + ] ++ lib.optionals config.simulacrum [ + ./simulacrum/snakeoil-rpc-secret.nix ]; garageConfig = [ ./garage-gateway.nix @@ -49,6 +52,11 @@ in garageInternal = [ ./garage-internal.nix ]; garageExternal = [ ./garage-external.nix ]; }; + simulacrum = { + enable = true; + deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ]; + settings = ./simulacrum/test.nix; + }; }; links = { diff --git a/cluster/services/storage/garage.nix b/cluster/services/storage/garage.nix index a319ba1..93d0bcf 100644 --- a/cluster/services/storage/garage.nix +++ b/cluster/services/storage/garage.nix @@ -63,6 +63,8 @@ in }; systemd.services.garage = { + requires = [ "consul-ready.service" ]; + after = [ "consul-ready.service" ]; unitConfig = { RequiresMountsFor = [ cfg.settings.data_dir ]; }; diff --git a/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix b/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix new file mode 100644 index 0000000..501bf1b --- /dev/null +++ b/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix @@ -0,0 +1,3 @@ +{ + environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +} diff --git a/cluster/services/storage/simulacrum/test-data.nix b/cluster/services/storage/simulacrum/test-data.nix new file mode 100644 index 0000000..c688365 --- /dev/null +++ b/cluster/services/storage/simulacrum/test-data.nix @@ -0,0 +1,8 @@ +{ config, lib, ... }: + +{ + garage = lib.mkIf config.simulacrum { + keys.testkey = {}; + buckets.testbucket.allow.testKey = [ "read" "write" ]; + }; +} diff --git a/cluster/services/storage/simulacrum/test.nix b/cluster/services/storage/simulacrum/test.nix new file mode 100644 index 0000000..a1fcf17 --- /dev/null +++ b/cluster/services/storage/simulacrum/test.nix @@ -0,0 +1,105 @@ +{ cluster, lib, ... }: + +let + inherit (cluster.config.services.storage) nodes; + + firstGarageNode = lib.elemAt nodes.garage 0; +in + +{ + nodes = lib.genAttrs nodes.garage (node: { + services.garage = { + layout.initial = lib.genAttrs nodes.garage (_: { + capacity = lib.mkOverride 51 1000; + }); + }; + specialisation.modifiedLayout = { + inheritParentConfig = true; + configuration = { + services.garage = { + layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000; + }; + system.ascensions.garage-layout.incantations = lib.mkForce (i: [ + (i.runGarage '' + garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)" + garage layout apply --version 2 + '') + ]); + }; + }; + }); + + testScript = '' + import json + nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')] + garage1 = nodes[0] + + start_all() + + with subtest("should bootstrap new cluster"): + for node in nodes: + node.wait_for_unit("garage.service") + + for node in nodes: + node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") + + with subtest("should apply new layout with ascension"): + for node in nodes: + node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1') + + for node in nodes: + node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test") + + for node in nodes: + node.wait_until_succeeds("garage layout show | grep -w 2000") + assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") + assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l") + + consulConfig = json.loads(garage1.succeed("cat /etc/consul.json")) + addr = consulConfig["addresses"]["http"] + port = consulConfig["ports"]["http"] + setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}" + with subtest("should apply new layout from scratch"): + for node in nodes: + node.systemctl("stop garage.service") + node.succeed("rm -rf /var/lib/garage-metadata") + garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage") + + for node in nodes: + node.systemctl("start garage.service") + + for node in nodes: + node.wait_for_unit("garage.service") + + for node in nodes: + node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") + + for node in nodes: + node.wait_until_succeeds("garage layout show | grep -w 2000") + assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") + assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l") + + with subtest("should create specified buckets and keys"): + for node in nodes: + node.wait_for_unit("incandescence-garage.target") + garage1.succeed("garage key list | grep testkey") + garage1.succeed("garage bucket list | grep testbucket") + + with subtest("should delete unspecified keys"): + garage1.succeed("garage bucket create unwantedbucket") + garage1.succeed("garage key new --name unwantedkey") + garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true") + garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true") + garage1.succeed("systemctl restart garage.service") + garage1.wait_for_unit("incandescence-garage.target") + garage1.fail("garage key list | grep unwantedkey") + garage1.succeed("garage bucket list | grep unwantedbucket") + + with subtest("should delete unspecified buckets after grace period"): + garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1") + garage1.succeed("systemctl restart garage.service") + garage1.wait_for_unit("incandescence-garage.target") + garage1.fail("garage bucket list | grep unwantedbucket") + ''; +} +