cluster/services/storage: test in simulacrum

This commit is contained in:
Max Headroom 2024-07-23 02:47:13 +02:00
parent 59ff96697d
commit 7287fcb5db
5 changed files with 126 additions and 0 deletions

View file

@ -8,6 +8,7 @@ in
imports = [
./options.nix
./incandescence.nix
./simulacrum/test-data.nix
];
services.storage = {
@ -36,6 +37,8 @@ in
./garage.nix
./garage-options.nix
./garage-layout.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-rpc-secret.nix
];
garageConfig = [
./garage-gateway.nix
@ -49,6 +52,11 @@ in
garageInternal = [ ./garage-internal.nix ];
garageExternal = [ ./garage-external.nix ];
};
simulacrum = {
enable = true;
deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
settings = ./simulacrum/test.nix;
};
};
links = {

View file

@ -63,6 +63,8 @@ in
};
systemd.services.garage = {
requires = [ "consul-ready.service" ];
after = [ "consul-ready.service" ];
unitConfig = {
RequiresMountsFor = [ cfg.settings.data_dir ];
};

View file

@ -0,0 +1,3 @@
{
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
}

View file

@ -0,0 +1,8 @@
{ config, lib, ... }:
{
garage = lib.mkIf config.simulacrum {
keys.testkey = {};
buckets.testbucket.allow.testKey = [ "read" "write" ];
};
}

View file

@ -0,0 +1,105 @@
{ cluster, lib, ... }:
let
inherit (cluster.config.services.storage) nodes;
firstGarageNode = lib.elemAt nodes.garage 0;
in
{
nodes = lib.genAttrs nodes.garage (node: {
services.garage = {
layout.initial = lib.genAttrs nodes.garage (_: {
capacity = lib.mkOverride 51 1000;
});
};
specialisation.modifiedLayout = {
inheritParentConfig = true;
configuration = {
services.garage = {
layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
};
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
(i.runGarage ''
garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
garage layout apply --version 2
'')
]);
};
};
});
testScript = ''
import json
nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
garage1 = nodes[0]
start_all()
with subtest("should bootstrap new cluster"):
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
with subtest("should apply new layout with ascension"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
for node in nodes:
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
addr = consulConfig["addresses"]["http"]
port = consulConfig["ports"]["http"]
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
with subtest("should apply new layout from scratch"):
for node in nodes:
node.systemctl("stop garage.service")
node.succeed("rm -rf /var/lib/garage-metadata")
garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
for node in nodes:
node.systemctl("start garage.service")
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should create specified buckets and keys"):
for node in nodes:
node.wait_for_unit("incandescence-garage.target")
garage1.succeed("garage key list | grep testkey")
garage1.succeed("garage bucket list | grep testbucket")
with subtest("should delete unspecified keys"):
garage1.succeed("garage bucket create unwantedbucket")
garage1.succeed("garage key new --name unwantedkey")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage key list | grep unwantedkey")
garage1.succeed("garage bucket list | grep unwantedbucket")
with subtest("should delete unspecified buckets after grace period"):
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage bucket list | grep unwantedbucket")
'';
}