cluster/services/storage: test in simulacrum
This commit is contained in:
parent
59ff96697d
commit
7287fcb5db
5 changed files with 126 additions and 0 deletions
|
@ -8,6 +8,7 @@ in
|
|||
imports = [
|
||||
./options.nix
|
||||
./incandescence.nix
|
||||
./simulacrum/test-data.nix
|
||||
];
|
||||
|
||||
services.storage = {
|
||||
|
@ -36,6 +37,8 @@ in
|
|||
./garage.nix
|
||||
./garage-options.nix
|
||||
./garage-layout.nix
|
||||
] ++ lib.optionals config.simulacrum [
|
||||
./simulacrum/snakeoil-rpc-secret.nix
|
||||
];
|
||||
garageConfig = [
|
||||
./garage-gateway.nix
|
||||
|
@ -49,6 +52,11 @@ in
|
|||
garageInternal = [ ./garage-internal.nix ];
|
||||
garageExternal = [ ./garage-external.nix ];
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
|
||||
settings = ./simulacrum/test.nix;
|
||||
};
|
||||
};
|
||||
|
||||
links = {
|
||||
|
|
|
@ -63,6 +63,8 @@ in
|
|||
};
|
||||
|
||||
systemd.services.garage = {
|
||||
requires = [ "consul-ready.service" ];
|
||||
after = [ "consul-ready.service" ];
|
||||
unitConfig = {
|
||||
RequiresMountsFor = [ cfg.settings.data_dir ];
|
||||
};
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
}
|
8
cluster/services/storage/simulacrum/test-data.nix
Normal file
8
cluster/services/storage/simulacrum/test-data.nix
Normal file
|
@ -0,0 +1,8 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
garage = lib.mkIf config.simulacrum {
|
||||
keys.testkey = {};
|
||||
buckets.testbucket.allow.testKey = [ "read" "write" ];
|
||||
};
|
||||
}
|
105
cluster/services/storage/simulacrum/test.nix
Normal file
105
cluster/services/storage/simulacrum/test.nix
Normal file
|
@ -0,0 +1,105 @@
|
|||
{ cluster, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster.config.services.storage) nodes;
|
||||
|
||||
firstGarageNode = lib.elemAt nodes.garage 0;
|
||||
in
|
||||
|
||||
{
|
||||
nodes = lib.genAttrs nodes.garage (node: {
|
||||
services.garage = {
|
||||
layout.initial = lib.genAttrs nodes.garage (_: {
|
||||
capacity = lib.mkOverride 51 1000;
|
||||
});
|
||||
};
|
||||
specialisation.modifiedLayout = {
|
||||
inheritParentConfig = true;
|
||||
configuration = {
|
||||
services.garage = {
|
||||
layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
|
||||
(i.runGarage ''
|
||||
garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
|
||||
garage layout apply --version 2
|
||||
'')
|
||||
]);
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
|
||||
garage1 = nodes[0]
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap new cluster"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
with subtest("should apply new layout with ascension"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
|
||||
|
||||
for node in nodes:
|
||||
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
|
||||
addr = consulConfig["addresses"]["http"]
|
||||
port = consulConfig["ports"]["http"]
|
||||
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
|
||||
with subtest("should apply new layout from scratch"):
|
||||
for node in nodes:
|
||||
node.systemctl("stop garage.service")
|
||||
node.succeed("rm -rf /var/lib/garage-metadata")
|
||||
garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
|
||||
|
||||
for node in nodes:
|
||||
node.systemctl("start garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should create specified buckets and keys"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("incandescence-garage.target")
|
||||
garage1.succeed("garage key list | grep testkey")
|
||||
garage1.succeed("garage bucket list | grep testbucket")
|
||||
|
||||
with subtest("should delete unspecified keys"):
|
||||
garage1.succeed("garage bucket create unwantedbucket")
|
||||
garage1.succeed("garage key new --name unwantedkey")
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
|
||||
garage1.succeed("systemctl restart garage.service")
|
||||
garage1.wait_for_unit("incandescence-garage.target")
|
||||
garage1.fail("garage key list | grep unwantedkey")
|
||||
garage1.succeed("garage bucket list | grep unwantedbucket")
|
||||
|
||||
with subtest("should delete unspecified buckets after grace period"):
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
|
||||
garage1.succeed("systemctl restart garage.service")
|
||||
garage1.wait_for_unit("incandescence-garage.target")
|
||||
garage1.fail("garage bucket list | grep unwantedbucket")
|
||||
'';
|
||||
}
|
||||
|
Loading…
Reference in a new issue