The Simulacrum: Stage 6 #114
18 changed files with 273 additions and 282 deletions
|
@ -1,4 +1,8 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
|
||||
|
|
|
@ -7,6 +7,8 @@ in
|
|||
{
|
||||
imports = [
|
||||
./options.nix
|
||||
./incandescence.nix
|
||||
./simulacrum/test-data.nix
|
||||
];
|
||||
|
||||
services.storage = {
|
||||
|
@ -30,11 +32,15 @@ in
|
|||
heresy = [
|
||||
./heresy.nix
|
||||
./s3ql-upgrades.nix
|
||||
] ++ lib.optionals config.simulacrum [
|
||||
./simulacrum/snakeoil-heresy-passphrase.nix
|
||||
];
|
||||
garage = [
|
||||
./garage.nix
|
||||
./garage-options.nix
|
||||
./garage-layout.nix
|
||||
] ++ lib.optionals config.simulacrum [
|
||||
./simulacrum/snakeoil-rpc-secret.nix
|
||||
];
|
||||
garageConfig = [
|
||||
./garage-gateway.nix
|
||||
|
@ -48,6 +54,11 @@ in
|
|||
garageInternal = [ ./garage-internal.nix ];
|
||||
garageExternal = [ ./garage-external.nix ];
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
|
||||
settings = ./simulacrum/test.nix;
|
||||
};
|
||||
};
|
||||
|
||||
links = {
|
||||
|
@ -86,7 +97,10 @@ in
|
|||
};
|
||||
|
||||
garage = {
|
||||
keys.storage-prophet = {};
|
||||
keys.storage-prophet.locksmith = {
|
||||
nodes = [ "prophet" ];
|
||||
format = "s3ql";
|
||||
};
|
||||
buckets.storage-prophet = {
|
||||
allow.storage-prophet = [ "read" "write" ];
|
||||
};
|
||||
|
|
|
@ -8,7 +8,7 @@ in
|
|||
services.external-storage = {
|
||||
fileSystems.external = {
|
||||
mountpoint = "/srv/storage";
|
||||
authFile = ./secrets/external-storage-auth-${hostName}.age;
|
||||
locksmithSecret = "garage-storage-${hostName}";
|
||||
backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
|
||||
backendOptions = [ "disable-expect100" ];
|
||||
};
|
||||
|
|
|
@ -26,62 +26,6 @@ let
|
|||
sleep 1
|
||||
done
|
||||
}
|
||||
# FIXME: returns bogus empty string when one of the lists is empty
|
||||
diffAdded() {
|
||||
comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
|
||||
}
|
||||
diffRemoved() {
|
||||
comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
|
||||
}
|
||||
# FIXME: this does not handle list items with spaces
|
||||
listKeys() {
|
||||
garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
|
||||
}
|
||||
ensureKeys() {
|
||||
old="$(listKeys)"
|
||||
if [[ -z "$1" ]]; then
|
||||
for key in $old; do
|
||||
garage key delete --yes "$key"
|
||||
done
|
||||
elif [[ -z "$old" ]]; then
|
||||
for key in $1; do
|
||||
# don't print secret key
|
||||
garage key new --name "$key" >/dev/null
|
||||
echo Key "$key" was created.
|
||||
done
|
||||
else
|
||||
diffAdded "$old" "$1" | while read key; do
|
||||
# don't print secret key
|
||||
garage key new --name "$key" >/dev/null
|
||||
echo Key "$key" was created.
|
||||
done
|
||||
diffRemoved "$old" "$1" | while read key; do
|
||||
garage key delete --yes "$key"
|
||||
done
|
||||
fi
|
||||
}
|
||||
listBuckets() {
|
||||
garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
|
||||
}
|
||||
ensureBuckets() {
|
||||
old="$(listBuckets)"
|
||||
if [[ -z "$1" ]]; then
|
||||
for bucket in $old; do
|
||||
garage bucket delete --yes "$bucket"
|
||||
done
|
||||
elif [[ -z "$old" ]]; then
|
||||
for bucket in $1; do
|
||||
garage bucket create "$bucket"
|
||||
done
|
||||
else
|
||||
diffAdded "$old" "$1" | while read bucket; do
|
||||
garage bucket create "$bucket"
|
||||
done
|
||||
diffRemoved "$old" "$1" | while read bucket; do
|
||||
garage bucket delete --yes "$bucket"
|
||||
done
|
||||
fi
|
||||
}
|
||||
'';
|
||||
in
|
||||
|
||||
|
@ -118,7 +62,7 @@ in
|
|||
};
|
||||
format = mkOption {
|
||||
description = "Locksmith secret format.";
|
||||
type = enum [ "files" "aws" "envFile" ];
|
||||
type = enum [ "files" "aws" "envFile" "s3ql" ];
|
||||
default = "files";
|
||||
};
|
||||
owner = mkOption {
|
||||
|
@ -203,9 +147,7 @@ in
|
|||
garage layout apply --version 1
|
||||
'';
|
||||
};
|
||||
garage-apply = {
|
||||
distributed.enable = true;
|
||||
wantedBy = [ "garage.service" "multi-user.target" ];
|
||||
garage-ready = {
|
||||
wants = [ "garage.service" ];
|
||||
after = [ "garage.service" "garage-layout-init.service" ];
|
||||
path = [ config.services.garage.package ];
|
||||
|
@ -219,54 +161,78 @@ in
|
|||
script = ''
|
||||
source ${garageShellLibrary}
|
||||
waitForGarageOperational
|
||||
|
||||
ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
|
||||
ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
|
||||
|
||||
# key permissions
|
||||
${lib.pipe cfg.keys [
|
||||
(lib.mapAttrsToList (key: kCfg: ''
|
||||
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket permissions
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg:
|
||||
lib.mapAttrsToList (key: perms: ''
|
||||
garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
|
||||
garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
|
||||
'') bCfg.allow
|
||||
))
|
||||
lib.flatten
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket quotas
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg: ''
|
||||
garage bucket set-quotas '${bucket}' \
|
||||
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
|
||||
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket website access
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg: ''
|
||||
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
services.incandescence.providers.garage = {
|
||||
locksmith = true;
|
||||
wantedBy = [ "garage.service" "multi-user.target" ];
|
||||
partOf = [ "garage.service" ];
|
||||
wants = [ "garage-ready.service" ];
|
||||
after = [ "garage-ready.service" ];
|
||||
|
||||
packages = [
|
||||
config.services.garage.package
|
||||
];
|
||||
formulae = {
|
||||
key = {
|
||||
destroyAfterDays = 0;
|
||||
create = key: ''
|
||||
if [[ "$(garage key info ${lib.escapeShellArg key} 2>&1 >/dev/null)" == "Error: 0 matching keys" ]]; then
|
||||
# don't print secret key
|
||||
garage key new --name ${lib.escapeShellArg key} >/dev/null
|
||||
echo Key ${lib.escapeShellArg key} was created.
|
||||
else
|
||||
echo "Key already exists, assuming ownership"
|
||||
fi
|
||||
'';
|
||||
destroy = ''
|
||||
garage key delete --yes "$OBJECT"
|
||||
'';
|
||||
change = key: let
|
||||
kCfg = cfg.keys.${key};
|
||||
in ''
|
||||
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
|
||||
'';
|
||||
};
|
||||
bucket = {
|
||||
deps = [ "key" ];
|
||||
destroyAfterDays = 30;
|
||||
create = bucket: ''
|
||||
if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == "Error: Bucket not found" ]]; then
|
||||
garage bucket create ${lib.escapeShellArg bucket}
|
||||
else
|
||||
echo "Bucket already exists, assuming ownership"
|
||||
fi
|
||||
'';
|
||||
destroy = ''
|
||||
garage bucket delete --yes "$OBJECT"
|
||||
'';
|
||||
change = bucket: let
|
||||
bCfg = cfg.buckets.${bucket};
|
||||
in ''
|
||||
# permissions
|
||||
${lib.concatStringsSep "\n" (lib.flatten (
|
||||
lib.mapAttrsToList (key: perms: ''
|
||||
garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
|
||||
garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
|
||||
'') bCfg.allow
|
||||
))}
|
||||
|
||||
# quotas
|
||||
garage bucket set-quotas ${lib.escapeShellArg bucket} \
|
||||
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
|
||||
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
|
||||
|
||||
# website access
|
||||
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.locksmith.providers.garage = {
|
||||
wantedBy = [ "garage-apply.service" ];
|
||||
after = [ "garage-apply.service" ];
|
||||
secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
|
||||
common = {
|
||||
inherit (kCfg.locksmith) mode owner group nodes;
|
||||
|
@ -291,6 +257,12 @@ in
|
|||
AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
|
||||
AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
|
||||
'';
|
||||
s3ql = ''
|
||||
[s3c]
|
||||
storage-url: s3c4://
|
||||
backend-login: @@GARAGE_KEY_ID@@
|
||||
backend-password: @@GARAGE_SECRET_KEY@@
|
||||
'';
|
||||
}.${kCfg.locksmith.format};
|
||||
in {
|
||||
${key} = common // {
|
||||
|
|
|
@ -63,6 +63,8 @@ in
|
|||
};
|
||||
|
||||
systemd.services.garage = {
|
||||
requires = [ "consul-ready.service" ];
|
||||
after = [ "consul-ready.service" ];
|
||||
unitConfig = {
|
||||
RequiresMountsFor = [ cfg.settings.data_dir ];
|
||||
};
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
unitDescription = "Heresy Filesystem";
|
||||
authFile = ./secrets/heresy-encryption-key.age;
|
||||
underlay = "heresy";
|
||||
encrypt = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
10
cluster/services/storage/incandescence.nix
Normal file
10
cluster/services/storage/incandescence.nix
Normal file
|
@ -0,0 +1,10 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
incandescence.providers.garage = {
|
||||
objects = {
|
||||
key = lib.attrNames config.garage.keys;
|
||||
bucket = lib.attrNames config.garage.buckets;
|
||||
};
|
||||
};
|
||||
}
|
|
@ -1,11 +0,0 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
|
||||
2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
|
||||
-> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
|
||||
0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
|
||||
-> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
|
||||
LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
|
||||
--- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
|
||||
™ªg¬NÑ8´¨\K!p
«ï…7ù¶›käõ¯#ŒÏu›µ*{}Tþ0·|@ÉÿàE>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†‡4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
|
||||
µÿÂ9gÏi"Q
–ÿ
|
||||
™›Ù®à
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
environment.etc."dummy-secrets/storageAuth-heresy".text = ''
|
||||
[local]
|
||||
storage-url: local://
|
||||
fs-passphrase: simulacrum
|
||||
'';
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
}
|
8
cluster/services/storage/simulacrum/test-data.nix
Normal file
8
cluster/services/storage/simulacrum/test-data.nix
Normal file
|
@ -0,0 +1,8 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
garage = lib.mkIf config.simulacrum {
|
||||
keys.testkey = {};
|
||||
buckets.testbucket.allow.testKey = [ "read" "write" ];
|
||||
};
|
||||
}
|
105
cluster/services/storage/simulacrum/test.nix
Normal file
105
cluster/services/storage/simulacrum/test.nix
Normal file
|
@ -0,0 +1,105 @@
|
|||
{ cluster, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster.config.services.storage) nodes;
|
||||
|
||||
firstGarageNode = lib.elemAt nodes.garage 0;
|
||||
in
|
||||
|
||||
{
|
||||
nodes = lib.genAttrs nodes.garage (node: {
|
||||
services.garage = {
|
||||
layout.initial = lib.genAttrs nodes.garage (_: {
|
||||
capacity = lib.mkOverride 51 1000;
|
||||
});
|
||||
};
|
||||
specialisation.modifiedLayout = {
|
||||
inheritParentConfig = true;
|
||||
configuration = {
|
||||
services.garage = {
|
||||
layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
|
||||
(i.runGarage ''
|
||||
garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
|
||||
garage layout apply --version 2
|
||||
'')
|
||||
]);
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
|
||||
garage1 = nodes[0]
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap new cluster"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
with subtest("should apply new layout with ascension"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
|
||||
|
||||
for node in nodes:
|
||||
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
|
||||
addr = consulConfig["addresses"]["http"]
|
||||
port = consulConfig["ports"]["http"]
|
||||
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
|
||||
with subtest("should apply new layout from scratch"):
|
||||
for node in nodes:
|
||||
node.systemctl("stop garage.service")
|
||||
node.succeed("rm -rf /var/lib/garage-metadata")
|
||||
garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
|
||||
|
||||
for node in nodes:
|
||||
node.systemctl("start garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should create specified buckets and keys"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("incandescence-garage.target")
|
||||
garage1.succeed("garage key list | grep testkey")
|
||||
garage1.succeed("garage bucket list | grep testbucket")
|
||||
|
||||
with subtest("should delete unspecified keys"):
|
||||
garage1.succeed("garage bucket create unwantedbucket")
|
||||
garage1.succeed("garage key new --name unwantedkey")
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
|
||||
garage1.succeed("systemctl restart garage.service")
|
||||
garage1.wait_for_unit("incandescence-garage.target")
|
||||
garage1.fail("garage key list | grep unwantedkey")
|
||||
garage1.succeed("garage bucket list | grep unwantedbucket")
|
||||
|
||||
with subtest("should delete unspecified buckets after grace period"):
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
|
||||
garage1.succeed("systemctl restart garage.service")
|
||||
garage1.wait_for_unit("incandescence-garage.target")
|
||||
garage1.fail("garage bucket list | grep unwantedbucket")
|
||||
'';
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
{ cluster, lib, ... }:
|
||||
|
||||
{
|
||||
defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
${lib.pipe cluster.config.services.wireguard.nodes.mesh [
|
||||
|
|
|
@ -8,6 +8,8 @@ let
|
|||
cfgAge = config.age;
|
||||
|
||||
create = lib.flip lib.mapAttrs';
|
||||
|
||||
createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f;
|
||||
in
|
||||
|
||||
{
|
||||
|
@ -20,12 +22,17 @@ in
|
|||
fileSystems = lib.mkOption {
|
||||
description = "S3QL-based filesystems on top of CIFS mountpoints.";
|
||||
default = {};
|
||||
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: {
|
||||
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let
|
||||
authFile = if config.locksmithSecret != null then
|
||||
"/run/locksmith/${config.locksmithSecret}"
|
||||
else
|
||||
cfgAge.secrets."storageAuth-${name}".path;
|
||||
in {
|
||||
imports = [ ./filesystem-type.nix ];
|
||||
backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}";
|
||||
commonArgs = [
|
||||
"--cachedir" config.cacheDir
|
||||
"--authfile" cfgAge.secrets."storageAuth-${name}".path
|
||||
"--authfile" authFile
|
||||
] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]);
|
||||
}));
|
||||
};
|
||||
|
@ -57,9 +64,14 @@ in
|
|||
|
||||
age.secrets = lib.mkMerge [
|
||||
(create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; }))
|
||||
(create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
|
||||
(createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
|
||||
];
|
||||
|
||||
services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: {
|
||||
name = fs.unitName;
|
||||
value = [ fs.locksmithSecret ];
|
||||
});
|
||||
|
||||
fileSystems = create cfg.underlays (name: ul: {
|
||||
name = ul.mountpoint;
|
||||
value = {
|
||||
|
@ -97,7 +109,13 @@ in
|
|||
value = let
|
||||
isUnderlay = fs.underlay != null;
|
||||
|
||||
fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);
|
||||
backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
|
||||
|
||||
fsType = if isUnderlay then "local" else lib.head backendParts;
|
||||
|
||||
s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
|
||||
|
||||
s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
|
||||
|
||||
localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
|
||||
in {
|
||||
|
@ -120,8 +138,12 @@ in
|
|||
ExecStartPre = map lib.escapeShellArgs [
|
||||
[
|
||||
(let
|
||||
authFile = if fs.locksmithSecret != null then
|
||||
"/run/locksmith/${fs.locksmithSecret}"
|
||||
else
|
||||
cfgAge.secrets."storageAuth-${name}".path;
|
||||
mkfsEncrypted = ''
|
||||
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
|
||||
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
|
||||
| cut -d' ' -f2- \
|
||||
| ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
|
||||
'';
|
||||
|
@ -132,6 +154,11 @@ in
|
|||
|
||||
detectFs = {
|
||||
local = "test -e ${localBackendPath}/s3ql_metadata";
|
||||
s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
|
||||
export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
|
||||
export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
|
||||
${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
|
||||
'';
|
||||
}.${fsType} or null;
|
||||
in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
|
||||
if ! ${detectFs}; then
|
||||
|
|
|
@ -22,6 +22,10 @@ with lib;
|
|||
authFile = mkOption {
|
||||
type = types.path;
|
||||
};
|
||||
locksmithSecret = mkOption {
|
||||
type = with types; nullOr str;
|
||||
default = null;
|
||||
};
|
||||
cacheDir = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/cache/remote-storage/${name}";
|
||||
|
|
|
@ -15,12 +15,6 @@ in
|
|||
inherit (config) cluster;
|
||||
};
|
||||
|
||||
garage = pkgs.callPackage ./garage.nix {
|
||||
inherit (self'.packages) garage consul;
|
||||
inherit (self) nixosModules;
|
||||
inherit (config) cluster;
|
||||
};
|
||||
|
||||
ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
|
||||
inherit (self) nixosModules;
|
||||
previous = timeMachine.preUnstable;
|
||||
|
|
|
@ -1,155 +0,0 @@
|
|||
{ testers, nixosModules, cluster, garage, consul }:
|
||||
|
||||
testers.runNixOSTest {
|
||||
name = "garage";
|
||||
|
||||
imports = [
|
||||
./modules/consul.nix
|
||||
];
|
||||
|
||||
extraBaseModules.services.consul.package = consul;
|
||||
|
||||
nodes = let
|
||||
common = { config, lib, ... }: let
|
||||
inherit (config.networking) hostName primaryIPAddress;
|
||||
in {
|
||||
imports = lib.flatten [
|
||||
./modules/nixos/age-dummy-secrets
|
||||
./modules/nixos/age-dummy-secrets/options.nix
|
||||
nixosModules.ascensions
|
||||
nixosModules.systemd-extras
|
||||
nixosModules.consul-distributed-services
|
||||
nixosModules.port-magic
|
||||
cluster.config.services.storage.nixos.garage
|
||||
cluster.config.services.storage.nixos.garageInternal
|
||||
cluster.config.services.consul.nixos.ready
|
||||
];
|
||||
options.services.locksmith.providers = lib.mkOption {
|
||||
type = lib.types.raw;
|
||||
};
|
||||
config = {
|
||||
links.consulAgent = {
|
||||
protocol = "http";
|
||||
hostname = "consul";
|
||||
port = 8500;
|
||||
};
|
||||
_module.args = {
|
||||
depot.packages = { inherit garage; };
|
||||
cluster.config = {
|
||||
hostLinks.${hostName} = {
|
||||
garageRpc.tuple = "${primaryIPAddress}:3901";
|
||||
garageS3.tuple = "${primaryIPAddress}:8080";
|
||||
garageWeb.tuple = "${primaryIPAddress}:8081";
|
||||
};
|
||||
links.garageWeb.hostname = "web.garage.example.com";
|
||||
vars.meshNet.cidr = "192.168.0.0/16";
|
||||
};
|
||||
};
|
||||
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
networking.firewall.allowedTCPPorts = [ 3901 8080 ];
|
||||
services.garage = {
|
||||
layout.initial = lib.mkOverride 51 {
|
||||
garage1 = { zone = "dc1"; capacity = 1000; };
|
||||
garage2 = { zone = "dc1"; capacity = 1000; };
|
||||
garage3 = { zone = "dc1"; capacity = 1000; };
|
||||
};
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
|
||||
specialisation.modifiedLayout = {
|
||||
inheritParentConfig = true;
|
||||
configuration = {
|
||||
services.garage = {
|
||||
layout.initial = lib.mkForce {
|
||||
garage1 = { zone = "dc1"; capacity = 2000; };
|
||||
garage2 = { zone = "dc1"; capacity = 1000; };
|
||||
garage3 = { zone = "dc1"; capacity = 1000; };
|
||||
};
|
||||
keys.testKey.allow.createBucket = true;
|
||||
buckets = {
|
||||
bucket1 = {
|
||||
allow.testKey = [ "read" "write" ];
|
||||
quotas = {
|
||||
maxObjects = 300;
|
||||
maxSize = 400 * 1024 * 1024;
|
||||
};
|
||||
};
|
||||
bucket2 = {
|
||||
allow.testKey = [ "read" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
|
||||
(i.runGarage ''
|
||||
garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
|
||||
garage layout apply --version 2
|
||||
'')
|
||||
]);
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in {
|
||||
garage1.imports = [ common ];
|
||||
garage2.imports = [ common ];
|
||||
garage3.imports = [ common ];
|
||||
};
|
||||
|
||||
testScript = { nodes, ... }: /*python*/ ''
|
||||
nodes = [garage1, garage2, garage3]
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap new cluster"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
with subtest("should apply new layout with ascension"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
|
||||
|
||||
for node in nodes:
|
||||
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should apply new layout from scratch"):
|
||||
for node in nodes:
|
||||
node.systemctl("stop garage.service")
|
||||
node.succeed("rm -rf /var/lib/garage-metadata")
|
||||
|
||||
for node in nodes:
|
||||
node.systemctl("start garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should create specified buckets and keys"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
|
||||
garage1.succeed("garage key list | grep testKey")
|
||||
garage1.succeed("garage bucket list | grep bucket1")
|
||||
garage1.succeed("garage bucket list | grep bucket2")
|
||||
|
||||
with subtest("should delete unspecified buckets and keys"):
|
||||
garage1.succeed("garage bucket create unwantedbucket")
|
||||
garage1.succeed("garage key new --name unwantedkey")
|
||||
garage1.succeed("systemctl restart garage-apply.service")
|
||||
|
||||
garage1.fail("garage key list | grep unwantedkey")
|
||||
garage1.fail("garage bucket list | grep unwantedbucket")
|
||||
'';
|
||||
}
|
|
@ -10,6 +10,9 @@ testers.runNixOSTest {
|
|||
nixosModules.systemd-extras
|
||||
./modules/nixos/age-dummy-secrets
|
||||
./modules/nixos/age-dummy-secrets/options.nix
|
||||
{
|
||||
options.services.locksmith = lib.mkSinkUndeclaredOptions { };
|
||||
}
|
||||
];
|
||||
|
||||
_module.args.depot.packages = { inherit (previous.packages.${system}) s3ql; };
|
||||
|
|
Loading…
Reference in a new issue