privatevoid.net/depot
1
1
Fork 0
Code Issues 23 Pull requests 2 Projects 3 Releases Packages Wiki Activity Actions

Merge pull request 'The Simulacrum: Stage 6' (#114) from pr-simulacrum-stage-6 into master

Browse source 
Reviewed-on: https://forge.privatevoid.net///privatevoid.net/depot/pulls/114
This commit is contained in:
Max Headroom 2024-08-17 01:13:16 +03:00
commit 5269b1f638
18 changed files with 273 additions and 282 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cluster/services/consul/test.nix
View file

@ -1,4 +1,8 @@
{ lib, ... }:
{
defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
testScript = ''
import json

16
cluster/services/storage/default.nix
View file

@ -7,6 +7,8 @@ in
{
imports = [
./options.nix
./incandescence.nix
./simulacrum/test-data.nix
];
services.storage = {
@ -30,11 +32,15 @@ in
heresy = [
./heresy.nix
./s3ql-upgrades.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-heresy-passphrase.nix
];
garage = [
./garage.nix
./garage-options.nix
./garage-layout.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-rpc-secret.nix
];
garageConfig = [
./garage-gateway.nix
@ -48,6 +54,11 @@ in
garageInternal = [ ./garage-internal.nix ];
garageExternal = [ ./garage-external.nix ];
};
simulacrum = {
enable = true;
deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
settings = ./simulacrum/test.nix;
};
};
links = {
@ -86,7 +97,10 @@ in
};
garage = {
keys.storage-prophet = {};
keys.storage-prophet.locksmith = {
nodes = [ "prophet" ];
format = "s3ql";
};
buckets.storage-prophet = {
allow.storage-prophet = [ "read" "write" ];
};

2
cluster/services/storage/external.nix
View file

@ -8,7 +8,7 @@ in
services.external-storage = {
fileSystems.external = {
mountpoint = "/srv/storage";
authFile = ./secrets/external-storage-auth-${hostName}.age;
locksmithSecret = "garage-storage-${hostName}";
backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
backendOptions = [ "disable-expect100" ];
};

178
cluster/services/storage/garage-options.nix
View file

@ -26,62 +26,6 @@ let
sleep 1
done
}
# FIXME: returns bogus empty string when one of the lists is empty
diffAdded() {
comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
}
diffRemoved() {
comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
}
# FIXME: this does not handle list items with spaces
listKeys() {
garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
}
ensureKeys() {
old="$(listKeys)"
if [[ -z "$1" ]]; then
for key in $old; do
garage key delete --yes "$key"
done
elif [[ -z "$old" ]]; then
for key in $1; do
# don't print secret key
garage key new --name "$key" >/dev/null
echo Key "$key" was created.
done
else
diffAdded "$old" "$1" | while read key; do
# don't print secret key
garage key new --name "$key" >/dev/null
echo Key "$key" was created.
done
diffRemoved "$old" "$1" | while read key; do
garage key delete --yes "$key"
done
fi
}
listBuckets() {
garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
}
ensureBuckets() {
old="$(listBuckets)"
if [[ -z "$1" ]]; then
for bucket in $old; do
garage bucket delete --yes "$bucket"
done
elif [[ -z "$old" ]]; then
for bucket in $1; do
garage bucket create "$bucket"
done
else
diffAdded "$old" "$1" | while read bucket; do
garage bucket create "$bucket"
done
diffRemoved "$old" "$1" | while read bucket; do
garage bucket delete --yes "$bucket"
done
fi
}
'';
in
@ -118,7 +62,7 @@ in
};
format = mkOption {
description = "Locksmith secret format.";
type = enum [ "files" "aws" "envFile" ];
type = enum [ "files" "aws" "envFile" "s3ql" ];
default = "files";
};
owner = mkOption {
@ -203,9 +147,7 @@ in
garage layout apply --version 1
'';
};
garage-apply = {
distributed.enable = true;
wantedBy = [ "garage.service" "multi-user.target" ];
garage-ready = {
wants = [ "garage.service" ];
after = [ "garage.service" "garage-layout-init.service" ];
path = [ config.services.garage.package ];
@ -219,54 +161,78 @@ in
script = ''
source ${garageShellLibrary}
waitForGarageOperational
ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
# key permissions
${lib.pipe cfg.keys [
(lib.mapAttrsToList (key: kCfg: ''
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
''))
(lib.concatStringsSep "\n")
]}
# bucket permissions
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg:
lib.mapAttrsToList (key: perms: ''
garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
'') bCfg.allow
))
lib.flatten
(lib.concatStringsSep "\n")
]}
# bucket quotas
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg: ''
garage bucket set-quotas '${bucket}' \
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
''))
(lib.concatStringsSep "\n")
]}
# bucket website access
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg: ''
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
''))
(lib.concatStringsSep "\n")
]}
'';
};
};
services.incandescence.providers.garage = {
locksmith = true;
wantedBy = [ "garage.service" "multi-user.target" ];
partOf = [ "garage.service" ];
wants = [ "garage-ready.service" ];
after = [ "garage-ready.service" ];
packages = [
config.services.garage.package
];
formulae = {
key = {
destroyAfterDays = 0;
create = key: ''
if [[ "$(garage key info ${lib.escapeShellArg key} 2>&1 >/dev/null)" == "Error: 0 matching keys" ]]; then
# don't print secret key
garage key new --name ${lib.escapeShellArg key} >/dev/null
echo Key ${lib.escapeShellArg key} was created.
else
echo "Key already exists, assuming ownership"
fi
'';
destroy = ''
garage key delete --yes "$OBJECT"
'';
change = key: let
kCfg = cfg.keys.${key};
in ''
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
'';
};
bucket = {
deps = [ "key" ];
destroyAfterDays = 30;
create = bucket: ''
if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == "Error: Bucket not found" ]]; then
garage bucket create ${lib.escapeShellArg bucket}
else
echo "Bucket already exists, assuming ownership"
fi
'';
destroy = ''
garage bucket delete --yes "$OBJECT"
'';
change = bucket: let
bCfg = cfg.buckets.${bucket};
in ''
# permissions
${lib.concatStringsSep "\n" (lib.flatten (
lib.mapAttrsToList (key: perms: ''
garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
'') bCfg.allow
))}
# quotas
garage bucket set-quotas ${lib.escapeShellArg bucket} \
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
# website access
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
'';
};
};
};
services.locksmith.providers.garage = {
wantedBy = [ "garage-apply.service" ];
after = [ "garage-apply.service" ];
secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
common = {
inherit (kCfg.locksmith) mode owner group nodes;
@ -291,6 +257,12 @@ in
AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
'';
s3ql = ''
[s3c]
storage-url: s3c4://
backend-login: @@GARAGE_KEY_ID@@
backend-password: @@GARAGE_SECRET_KEY@@
'';
}.${kCfg.locksmith.format};
in {
${key} = common // {

2
cluster/services/storage/garage.nix
View file

@ -63,6 +63,8 @@ in
};
systemd.services.garage = {
requires = [ "consul-ready.service" ];
after = [ "consul-ready.service" ];
unitConfig = {
RequiresMountsFor = [ cfg.settings.data_dir ];
};

1
cluster/services/storage/heresy.nix
View file

@ -11,6 +11,7 @@
unitDescription = "Heresy Filesystem";
authFile = ./secrets/heresy-encryption-key.age;
underlay = "heresy";
encrypt = true;
};
};
}

10
cluster/services/storage/incandescence.nix Normal file
View file

@ -0,0 +1,10 @@
{ config, lib, ... }:
{
incandescence.providers.garage = {
objects = {
key = lib.attrNames config.garage.keys;
bucket = lib.attrNames config.garage.buckets;
};
};
}

11
cluster/services/storage/secrets/external-storage-auth-prophet.age
View file

@ -1,11 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
-> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
-> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
--- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
™ªg¬NÑ 8´¨\K!p «ï…7ù¶›käõ¯#ŒÏu›µ*{}Tþ0·|@Éÿà E>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†‡4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
µÿÂ9gÏi"Q –ÿ
™›Ù®à

8
cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix Normal file
View file

@ -0,0 +1,8 @@
{
environment.etc."dummy-secrets/storageAuth-heresy".text = ''
[local]
storage-url: local://
fs-passphrase: simulacrum
'';
}

3
cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix Normal file
View file

@ -0,0 +1,3 @@
{
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
}

8
cluster/services/storage/simulacrum/test-data.nix Normal file
View file

@ -0,0 +1,8 @@
{ config, lib, ... }:
{
garage = lib.mkIf config.simulacrum {
keys.testkey = {};
buckets.testbucket.allow.testKey = [ "read" "write" ];
};
}

105
cluster/services/storage/simulacrum/test.nix Normal file
View file

@ -0,0 +1,105 @@
{ cluster, lib, ... }:
let
inherit (cluster.config.services.storage) nodes;
firstGarageNode = lib.elemAt nodes.garage 0;
in
{
nodes = lib.genAttrs nodes.garage (node: {
services.garage = {
layout.initial = lib.genAttrs nodes.garage (_: {
capacity = lib.mkOverride 51 1000;
});
};
specialisation.modifiedLayout = {
inheritParentConfig = true;
configuration = {
services.garage = {
layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
};
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
(i.runGarage ''
garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
garage layout apply --version 2
'')
]);
};
};
});
testScript = ''
import json
nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
garage1 = nodes[0]
start_all()
with subtest("should bootstrap new cluster"):
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
with subtest("should apply new layout with ascension"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
for node in nodes:
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
addr = consulConfig["addresses"]["http"]
port = consulConfig["ports"]["http"]
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
with subtest("should apply new layout from scratch"):
for node in nodes:
node.systemctl("stop garage.service")
node.succeed("rm -rf /var/lib/garage-metadata")
garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
for node in nodes:
node.systemctl("start garage.service")
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should create specified buckets and keys"):
for node in nodes:
node.wait_for_unit("incandescence-garage.target")
garage1.succeed("garage key list | grep testkey")
garage1.succeed("garage bucket list | grep testbucket")
with subtest("should delete unspecified keys"):
garage1.succeed("garage bucket create unwantedbucket")
garage1.succeed("garage key new --name unwantedkey")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage key list | grep unwantedkey")
garage1.succeed("garage bucket list | grep unwantedbucket")
with subtest("should delete unspecified buckets after grace period"):
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage bucket list | grep unwantedbucket")
'';
}

2
cluster/services/wireguard/test.nix
View file

@ -1,6 +1,8 @@
{ cluster, lib, ... }:
{
defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
testScript = ''
start_all()
${lib.pipe cluster.config.services.wireguard.nodes.mesh [

37
modules/external-storage/default.nix
View file

@ -8,6 +8,8 @@ let
cfgAge = config.age;
create = lib.flip lib.mapAttrs';
createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f;
in
{
@ -20,12 +22,17 @@ in
fileSystems = lib.mkOption {
description = "S3QL-based filesystems on top of CIFS mountpoints.";
default = {};
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: {
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let
authFile = if config.locksmithSecret != null then
"/run/locksmith/${config.locksmithSecret}"
else
cfgAge.secrets."storageAuth-${name}".path;
in {
imports = [ ./filesystem-type.nix ];
backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}";
commonArgs = [
"--cachedir" config.cacheDir
"--authfile" cfgAge.secrets."storageAuth-${name}".path
"--authfile" authFile
] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]);
}));
};
@ -57,9 +64,14 @@ in
age.secrets = lib.mkMerge [
(create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; }))
(create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
(createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
];
services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: {
name = fs.unitName;
value = [ fs.locksmithSecret ];
});
fileSystems = create cfg.underlays (name: ul: {
name = ul.mountpoint;
value = {
@ -97,7 +109,13 @@ in
value = let
isUnderlay = fs.underlay != null;
fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);
backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
fsType = if isUnderlay then "local" else lib.head backendParts;
s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
in {
@ -120,8 +138,12 @@ in
ExecStartPre = map lib.escapeShellArgs [
[
(let
authFile = if fs.locksmithSecret != null then
"/run/locksmith/${fs.locksmithSecret}"
else
cfgAge.secrets."storageAuth-${name}".path;
mkfsEncrypted = ''
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
| cut -d' ' -f2- \
| ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
'';
@ -132,6 +154,11 @@ in
detectFs = {
local = "test -e ${localBackendPath}/s3ql_metadata";
s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
'';
}.${fsType} or null;
in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
if ! ${detectFs}; then

4
modules/external-storage/filesystem-type.nix
View file

@ -22,6 +22,10 @@ with lib;
authFile = mkOption {
type = types.path;
};
locksmithSecret = mkOption {
type = with types; nullOr str;
default = null;
};
cacheDir = mkOption {
type = types.path;
default = "/var/cache/remote-storage/${name}";

6
packages/checks/default.nix
View file

@ -15,12 +15,6 @@ in
inherit (config) cluster;
};
garage = pkgs.callPackage ./garage.nix {
inherit (self'.packages) garage consul;
inherit (self) nixosModules;
inherit (config) cluster;
};
ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
inherit (self) nixosModules;
previous = timeMachine.preUnstable;

155
packages/checks/garage.nix
View file

@ -1,155 +0,0 @@
{ testers, nixosModules, cluster, garage, consul }:
testers.runNixOSTest {
name = "garage";
imports = [
./modules/consul.nix
];
extraBaseModules.services.consul.package = consul;
nodes = let
common = { config, lib, ... }: let
inherit (config.networking) hostName primaryIPAddress;
in {
imports = lib.flatten [
./modules/nixos/age-dummy-secrets
./modules/nixos/age-dummy-secrets/options.nix
nixosModules.ascensions
nixosModules.systemd-extras
nixosModules.consul-distributed-services
nixosModules.port-magic
cluster.config.services.storage.nixos.garage
cluster.config.services.storage.nixos.garageInternal
cluster.config.services.consul.nixos.ready
];
options.services.locksmith.providers = lib.mkOption {
type = lib.types.raw;
};
config = {
links.consulAgent = {
protocol = "http";
hostname = "consul";
port = 8500;
};
_module.args = {
depot.packages = { inherit garage; };
cluster.config = {
hostLinks.${hostName} = {
garageRpc.tuple = "${primaryIPAddress}:3901";
garageS3.tuple = "${primaryIPAddress}:8080";
garageWeb.tuple = "${primaryIPAddress}:8081";
};
links.garageWeb.hostname = "web.garage.example.com";
vars.meshNet.cidr = "192.168.0.0/16";
};
};
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
networking.firewall.allowedTCPPorts = [ 3901 8080 ];
services.garage = {
layout.initial = lib.mkOverride 51 {
garage1 = { zone = "dc1"; capacity = 1000; };
garage2 = { zone = "dc1"; capacity = 1000; };
garage3 = { zone = "dc1"; capacity = 1000; };
};
};
system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
specialisation.modifiedLayout = {
inheritParentConfig = true;
configuration = {
services.garage = {
layout.initial = lib.mkForce {
garage1 = { zone = "dc1"; capacity = 2000; };
garage2 = { zone = "dc1"; capacity = 1000; };
garage3 = { zone = "dc1"; capacity = 1000; };
};
keys.testKey.allow.createBucket = true;
buckets = {
bucket1 = {
allow.testKey = [ "read" "write" ];
quotas = {
maxObjects = 300;
maxSize = 400 * 1024 * 1024;
};
};
bucket2 = {
allow.testKey = [ "read" ];
};
};
};
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
(i.runGarage ''
garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
garage layout apply --version 2
'')
]);
};
};
};
};
in {
garage1.imports = [ common ];
garage2.imports = [ common ];
garage3.imports = [ common ];
};
testScript = { nodes, ... }: /*python*/ ''
nodes = [garage1, garage2, garage3]
start_all()
with subtest("should bootstrap new cluster"):
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
with subtest("should apply new layout with ascension"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
for node in nodes:
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should apply new layout from scratch"):
for node in nodes:
node.systemctl("stop garage.service")
node.succeed("rm -rf /var/lib/garage-metadata")
for node in nodes:
node.systemctl("start garage.service")
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should create specified buckets and keys"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
garage1.succeed("garage key list | grep testKey")
garage1.succeed("garage bucket list | grep bucket1")
garage1.succeed("garage bucket list | grep bucket2")
with subtest("should delete unspecified buckets and keys"):
garage1.succeed("garage bucket create unwantedbucket")
garage1.succeed("garage key new --name unwantedkey")
garage1.succeed("systemctl restart garage-apply.service")
garage1.fail("garage key list | grep unwantedkey")
garage1.fail("garage bucket list | grep unwantedbucket")
'';
}

3
packages/checks/s3ql-upgrade.nix
View file

@ -10,6 +10,9 @@ testers.runNixOSTest {
nixosModules.systemd-extras
./modules/nixos/age-dummy-secrets
./modules/nixos/age-dummy-secrets/options.nix
{
options.services.locksmith = lib.mkSinkUndeclaredOptions { };
}
];
_module.args.depot.packages = { inherit (previous.packages.${system}) s3ql; };