privatevoid.net/depot
1
1
Fork 0
Code Issues 18 Pull requests 2 Projects 3 Releases Packages Wiki Activity Actions

WIP: The Simulacrum #106

Closed
max wants to merge 15 commits from pr-full-test into master
pull from: pr-full-test
merge into: privatevoid.net:master
Conversation 1 Commits 15 Files changed 22 +328 -306
22 changed files with 328 additions and 306 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
cluster/catalog/secrets.nix
View file

@ -46,6 +46,7 @@ in
};
}) // (if secretConfig.shared then let
secretFile = "${svcName}-${secretName}.age";
snakeoilFile = "${svcName}-${secretName}-snakeoil.txt";
in {
editSecret = {
description = "Edit this secret";
@ -54,15 +55,31 @@ in
agenix -e '${secretFile}'
'';
};
} else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
secretFile = "${svcName}-${secretName}-${node}.age";
in {
description = "Edit this secret for '${node}'";
command = ''
${setupCommands secretFile [ node ]}
agenix -e '${secretFile}'
'';
})));
editSnakeoil = {
description = "Edit this secret's snakeoil";
command = ''
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
'';
};
} else lib.mkMerge [
(lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
secretFile = "${svcName}-${secretName}-${node}.age";
in {
description = "Edit this secret for '${node}'";
command = ''
${setupCommands secretFile [ node ]}
agenix -e '${secretFile}'
'';
})))
(lib.mapAttrs' (name: lib.nameValuePair "editSnakeoilInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
snakeoilFile = "${svcName}-${secretName}-${node}-snakeoil.txt";
in {
description = "Edit this secret's snakeoil for '${node}'";
command = ''
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
'';
})))
]);
};
}) svcConfig.secrets))
lib.concatLists

13
cluster/secrets/forge-dbCredentials.age
View file

@ -1,13 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
-> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
-> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
-> Ovax-grease ^1$]}H G4 FpDF XKHkj{
IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
--- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
ÜA³×oÈð:±­‹`ÜVd±å(Kät:fk¼’}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69 i›l`ÛÆJKwAè8­y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ý’w Ê
ÆÜßÌZî!^þRˆéÿv­¾…ïk‹Êp»ÛPÌ)ý̆ÍpÓV5²F΄ÆÚÙÚÞhBÇ»ß b# Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè‹}€ß ƒ¬E}~ZHûjmyq{òxŠ–Éôß<C3B4>"”éÀ´C#šójÿÐ.ò§y Ô£¸v¦ <0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që

7
cluster/services/consul/agent.nix
View file

@ -10,6 +10,13 @@ let
in
{
system.extraIncantations = {
runConsul = i: script: i.execShellWith [ config.services.consul.package ] ''
export CONSUL_HTTP_ADDR='${config.links.consulAgent.tuple}'
${script}
'';
};
links.consulAgent.protocol = "http";
services.consul = {

10
cluster/services/dns/default.nix
View file

@ -73,6 +73,16 @@ in
};
};
patroni = {
databases.acmedns = {};
users.acmedns = {
locksmith = {
nodes = config.services.dns.nodes.authoritative;
format = "envFile";
};
};
};
dns.records = {
securedns.consulService = "securedns";
"acme-dns-challenge.internal".consulService = "acme-dns";

9
cluster/services/forge/default.nix
View file

@ -13,7 +13,6 @@
nodes = server;
owner = "forgejo";
};
dbCredentials.nodes = server;
};
};
@ -23,6 +22,14 @@
forge.target = config.hostLinks.${host}.forge.url;
};
patroni = config.lib.forService "forge" {
databases.forge = {};
users.forge.locksmith = {
nodes = config.services.forge.nodes.server;
format = "raw";
};
};
garage = config.lib.forService "forge" {
keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
buckets.forgejo.allow.forgejo = [ "read" "write" ];

3
cluster/services/forge/server.nix
View file

@ -26,6 +26,7 @@ in
services.locksmith.waitForSecrets.forgejo = [
"garage-forgejo-id"
"garage-forgejo-secret"
"patroni-forge"
];
services.forgejo = {
@ -39,7 +40,7 @@ in
inherit (patroni) port;
name = "forge";
user = "forge";
passwordFile = secrets.dbCredentials.path;
passwordFile = "/run/locksmith/patroni-forge";
};
settings = {
DEFAULT = {

17
cluster/services/storage/default.nix
View file

@ -7,6 +7,8 @@ in
{
imports = [
./options.nix
./incandescence.nix
./simulacrum/test-data.nix
];
services.storage = {
@ -30,11 +32,16 @@ in
heresy = [
./heresy.nix
./s3ql-upgrades.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-heresy-passphrase.nix
];
garage = [
./garage.nix
./garage-options.nix
./garage-layout.nix
./incandescence-ascensions.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-rpc-secret.nix
];
garageConfig = [
./garage-gateway.nix
@ -48,6 +55,11 @@ in
garageInternal = [ ./garage-internal.nix ];
garageExternal = [ ./garage-external.nix ];
};
simulacrum = {
enable = true;
deps = [ "consul" "locksmith" "incandescence" "patroni" "ways" ];
settings = ./test.nix;
};
};
links = {
@ -86,7 +98,10 @@ in
};
garage = {
keys.storage-prophet = {};
keys.storage-prophet.locksmith = {
nodes = [ "prophet" ];
format = "s3ql";
};
buckets.storage-prophet = {
allow.storage-prophet = [ "read" "write" ];
};

2
cluster/services/storage/external.nix
View file

@ -8,7 +8,7 @@ in
services.external-storage = {
fileSystems.external = {
mountpoint = "/srv/storage";
authFile = ./secrets/external-storage-auth-${hostName}.age;
locksmithSecret = "garage-storage-${hostName}";
backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
backendOptions = [ "disable-expect100" ];
};

170
cluster/services/storage/garage-options.nix
View file

@ -26,62 +26,6 @@ let
sleep 1
done
}
# FIXME: returns bogus empty string when one of the lists is empty
diffAdded() {
comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
}
diffRemoved() {
comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
}
# FIXME: this does not handle list items with spaces
listKeys() {
garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
}
ensureKeys() {
old="$(listKeys)"
if [[ -z "$1" ]]; then
for key in $old; do
garage key delete --yes "$key"
done
elif [[ -z "$old" ]]; then
for key in $1; do
# don't print secret key
garage key new --name "$key" >/dev/null
echo Key "$key" was created.
done
else
diffAdded "$old" "$1" | while read key; do
# don't print secret key
garage key new --name "$key" >/dev/null
echo Key "$key" was created.
done
diffRemoved "$old" "$1" | while read key; do
garage key delete --yes "$key"
done
fi
}
listBuckets() {
garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
}
ensureBuckets() {
old="$(listBuckets)"
if [[ -z "$1" ]]; then
for bucket in $old; do
garage bucket delete --yes "$bucket"
done
elif [[ -z "$old" ]]; then
for bucket in $1; do
garage bucket create "$bucket"
done
else
diffAdded "$old" "$1" | while read bucket; do
garage bucket create "$bucket"
done
diffRemoved "$old" "$1" | while read bucket; do
garage bucket delete --yes "$bucket"
done
fi
}
'';
in
@ -118,7 +62,7 @@ in
};
format = mkOption {
description = "Locksmith secret format.";
type = enum [ "files" "aws" "envFile" ];
type = enum [ "files" "aws" "envFile" "s3ql" ];
default = "files";
};
owner = mkOption {
@ -203,9 +147,7 @@ in
garage layout apply --version 1
'';
};
garage-apply = {
distributed.enable = true;
wantedBy = [ "garage.service" "multi-user.target" ];
garage-ready = {
wants = [ "garage.service" ];
after = [ "garage.service" "garage-layout-init.service" ];
path = [ config.services.garage.package ];
@ -219,54 +161,70 @@ in
script = ''
source ${garageShellLibrary}
waitForGarageOperational
ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
# key permissions
${lib.pipe cfg.keys [
(lib.mapAttrsToList (key: kCfg: ''
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
''))
(lib.concatStringsSep "\n")
]}
# bucket permissions
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg:
lib.mapAttrsToList (key: perms: ''
garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
'') bCfg.allow
))
lib.flatten
(lib.concatStringsSep "\n")
]}
# bucket quotas
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg: ''
garage bucket set-quotas '${bucket}' \
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
''))
(lib.concatStringsSep "\n")
]}
# bucket website access
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg: ''
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
''))
(lib.concatStringsSep "\n")
]}
'';
};
};
services.incandescence.providers.garage = {
locksmith = true;
wantedBy = [ "garage.service" "multi-user.target" ];
partOf = [ "garage.service" ];
wants = [ "garage-ready.service" ];
after = [ "garage-ready.service" ];
packages = [
config.services.garage.package
];
formulae = {
key = {
destroyAfterDays = 0;
create = key: ''
# don't print secret key
garage key new --name ${lib.escapeShellArg key} >/dev/null
echo Key ${lib.escapeShellArg key} was created.
'';
destroy = ''
garage key delete --yes "$OBJECT"
'';
change = key: let
kCfg = cfg.keys.${key};
in ''
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
'';
};
bucket = {
deps = [ "key" ];
destroyAfterDays = 30;
create = bucket: ''
garage bucket create ${lib.escapeShellArg bucket}
'';
destroy = ''
garage bucket delete --yes "$OBJECT"
'';
change = bucket: let
bCfg = cfg.buckets.${bucket};
in ''
# permissions
${lib.concatStringsSep "\n" (lib.flatten (
lib.mapAttrsToList (key: perms: ''
garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
'') bCfg.allow
))}
# quotas
garage bucket set-quotas ${lib.escapeShellArg bucket} \
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
# website access
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
'';
};
};
};
services.locksmith.providers.garage = {
wantedBy = [ "garage-apply.service" ];
after = [ "garage-apply.service" ];
secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
common = {
inherit (kCfg.locksmith) mode owner group nodes;
@ -291,6 +249,12 @@ in
AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
'';
s3ql = ''
[s3c]
storage-url: s3c4://
backend-login: @@GARAGE_KEY_ID@@
backend-password: @@GARAGE_SECRET_KEY@@
'';
}.${kCfg.locksmith.format};
in {
${key} = common // {

2
cluster/services/storage/garage.nix
View file

@ -63,6 +63,8 @@ in
};
systemd.services.garage = {
requires = [ "consul-ready.service" ];
after = [ "consul-ready.service" ];
unitConfig = {
RequiresMountsFor = [ cfg.settings.data_dir ];
};

1
cluster/services/storage/heresy.nix
View file

@ -11,6 +11,7 @@
unitDescription = "Heresy Filesystem";
authFile = ./secrets/heresy-encryption-key.age;
underlay = "heresy";
encrypt = true;
};
};
}

18
cluster/services/storage/incandescence-ascensions.nix Normal file
View file

@ -0,0 +1,18 @@
{ config, lib, ... }:
{
system.ascensions = {
incandescence-garage = lib.mkIf (config.services.incandescence.providers ? garage) {
incantations = i: [
(i.runGarage /*bash*/ ''
garage bucket list | tail -n +2 | cut -d' ' -f3 | while read bucket; do
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/bucket/$1/alive" true''} "$bucket"
done
garage key list | tail -n +2 | cut -d' ' -f5 | while read key; do
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/key/$1/alive" true''} "$key"
done
'')
];
};
};
}

10
cluster/services/storage/incandescence.nix Normal file
View file

@ -0,0 +1,10 @@
{ config, lib, ... }:
{
incandescence.providers.garage = {
objects = {
key = lib.attrNames config.garage.keys;
bucket = lib.attrNames config.garage.buckets;
};
};
}

11
cluster/services/storage/secrets/external-storage-auth-prophet.age
View file

@ -1,11 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
-> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
-> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
--- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
™ªg¬NÑ 8´¨\K!p «ï…7ù¶›käõ¯#ŒÏu›µ*{}Tþ0·|@Éÿà E>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†‡4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
µÿÂ9gÏi"Q –ÿ
™›Ù®à

8
cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix Normal file
View file

@ -0,0 +1,8 @@
{
environment.etc."dummy-secrets/storageAuth-heresy".text = ''
[local]
storage-url: local://
fs-passphrase: simulacrum
'';
}

3
cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix Normal file
View file

@ -0,0 +1,3 @@
{
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
}

8
cluster/services/storage/simulacrum/test-data.nix Normal file
View file

@ -0,0 +1,8 @@
{ config, lib, ... }:
{
garage = lib.mkIf config.simulacrum {
keys.testkey = {};
buckets.testbucket.allow.testKey = [ "read" "write" ];
};
}

105
cluster/services/storage/test.nix Normal file
View file

@ -0,0 +1,105 @@
{ cluster, lib, ... }:
let
inherit (cluster.config.services.storage) nodes;
firstGarageNode = lib.elemAt nodes.garage 0;
in
{
nodes = lib.genAttrs nodes.garage (node: {
services.garage = {
layout.initial = lib.genAttrs nodes.garage (_: {
capacity = lib.mkOverride 51 1000;
});
};
specialisation.modifiedLayout = {
inheritParentConfig = true;
configuration = {
services.garage = {
layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
};
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
(i.runGarage ''
garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
garage layout apply --version 2
'')
]);
};
};
});
testScript = ''
import json
nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
garage1 = nodes[0]
start_all()
with subtest("should bootstrap new cluster"):
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
with subtest("should apply new layout with ascension"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
for node in nodes:
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
addr = consulConfig["addresses"]["http"]
port = consulConfig["ports"]["http"]
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
with subtest("should apply new layout from scratch"):
for node in nodes:
node.systemctl("stop garage.service")
node.succeed("rm -rf /var/lib/garage-metadata")
garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
for node in nodes:
node.systemctl("start garage.service")
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should create specified buckets and keys"):
for node in nodes:
node.wait_for_unit("incandescence-garage.target")
garage1.succeed("garage key list | grep testkey")
garage1.succeed("garage bucket list | grep testbucket")
with subtest("should delete unspecified keys"):
garage1.succeed("garage bucket create unwantedbucket")
garage1.succeed("garage key new --name unwantedkey")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage key list | grep unwantedkey")
garage1.succeed("garage bucket list | grep unwantedbucket")
with subtest("should delete unspecified buckets after grace period"):
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage bucket list | grep unwantedbucket")
'';
}

37
modules/external-storage/default.nix
View file

@ -8,6 +8,8 @@ let
cfgAge = config.age;
create = lib.flip lib.mapAttrs';
createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f;
in
{
@ -20,12 +22,17 @@ in
fileSystems = lib.mkOption {
description = "S3QL-based filesystems on top of CIFS mountpoints.";
default = {};
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: {
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let
authFile = if config.locksmithSecret != null then
"/run/locksmith/${config.locksmithSecret}"
else
cfgAge.secrets."storageAuth-${name}".path;
in {
imports = [ ./filesystem-type.nix ];
backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}";
commonArgs = [
"--cachedir" config.cacheDir
"--authfile" cfgAge.secrets."storageAuth-${name}".path
"--authfile" authFile
] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]);
}));
};
@ -57,9 +64,14 @@ in
age.secrets = lib.mkMerge [
(create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; }))
(create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
(createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
];
services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: {
name = fs.unitName;
value = [ fs.locksmithSecret ];
});
fileSystems = create cfg.underlays (name: ul: {
name = ul.mountpoint;
value = {
@ -97,7 +109,13 @@ in
value = let
isUnderlay = fs.underlay != null;
fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);
backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
fsType = if isUnderlay then "local" else lib.head backendParts;
s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
in {
@ -120,8 +138,12 @@ in
ExecStartPre = map lib.escapeShellArgs [
[
(let
authFile = if fs.locksmithSecret != null then
"/run/locksmith/${fs.locksmithSecret}"
else
cfgAge.secrets."storageAuth-${name}".path;
mkfsEncrypted = ''
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
| cut -d' ' -f2- \
| ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
'';
@ -132,6 +154,11 @@ in
detectFs = {
local = "test -e ${localBackendPath}/s3ql_metadata";
s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
'';
}.${fsType} or null;
in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
if ! ${detectFs}; then

4
modules/external-storage/filesystem-type.nix
View file

@ -22,6 +22,10 @@ with lib;
authFile = mkOption {
type = types.path;
};
locksmithSecret = mkOption {
type = with types; nullOr str;
default = null;
};
cacheDir = mkOption {
type = types.path;
default = "/var/cache/remote-storage/${name}";

6
packages/checks/default.nix
View file

@ -15,12 +15,6 @@ in
inherit (config) cluster;
};
garage = pkgs.callPackage ./garage.nix {
inherit (self'.packages) garage consul;
inherit (self) nixosModules;
inherit (config) cluster;
};
ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
inherit (self) nixosModules;
previous = timeMachine.preUnstable;

155
packages/checks/garage.nix
View file

@ -1,155 +0,0 @@
{ testers, nixosModules, cluster, garage, consul }:
testers.runNixOSTest {
name = "garage";
imports = [
./modules/consul.nix
];
extraBaseModules.services.consul.package = consul;
nodes = let
common = { config, lib, ... }: let
inherit (config.networking) hostName primaryIPAddress;
in {
imports = lib.flatten [
./modules/nixos/age-dummy-secrets
./modules/nixos/age-dummy-secrets/options.nix
nixosModules.ascensions
nixosModules.systemd-extras
nixosModules.consul-distributed-services
nixosModules.port-magic
cluster.config.services.storage.nixos.garage
cluster.config.services.storage.nixos.garageInternal
cluster.config.services.consul.nixos.ready
];
options.services.locksmith.providers = lib.mkOption {
type = lib.types.raw;
};
config = {
links.consulAgent = {
protocol = "http";
hostname = "consul";
port = 8500;
};
_module.args = {
depot.packages = { inherit garage; };
cluster.config = {
hostLinks.${hostName} = {
garageRpc.tuple = "${primaryIPAddress}:3901";
garageS3.tuple = "${primaryIPAddress}:8080";
garageWeb.tuple = "${primaryIPAddress}:8081";
};
links.garageWeb.hostname = "web.garage.example.com";
vars.meshNet.cidr = "192.168.0.0/16";
};
};
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
networking.firewall.allowedTCPPorts = [ 3901 8080 ];
services.garage = {
layout.initial = lib.mkOverride 51 {
garage1 = { zone = "dc1"; capacity = 1000; };
garage2 = { zone = "dc1"; capacity = 1000; };
garage3 = { zone = "dc1"; capacity = 1000; };
};
};
system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
specialisation.modifiedLayout = {
inheritParentConfig = true;
configuration = {
services.garage = {
layout.initial = lib.mkForce {
garage1 = { zone = "dc1"; capacity = 2000; };
garage2 = { zone = "dc1"; capacity = 1000; };
garage3 = { zone = "dc1"; capacity = 1000; };
};
keys.testKey.allow.createBucket = true;
buckets = {
bucket1 = {
allow.testKey = [ "read" "write" ];
quotas = {
maxObjects = 300;
maxSize = 400 * 1024 * 1024;
};
};
bucket2 = {
allow.testKey = [ "read" ];
};
};
};
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
(i.runGarage ''
garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
garage layout apply --version 2
'')
]);
};
};
};
};
in {
garage1.imports = [ common ];
garage2.imports = [ common ];
garage3.imports = [ common ];
};
testScript = { nodes, ... }: /*python*/ ''
nodes = [garage1, garage2, garage3]
start_all()
with subtest("should bootstrap new cluster"):
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
with subtest("should apply new layout with ascension"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
for node in nodes:
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should apply new layout from scratch"):
for node in nodes:
node.systemctl("stop garage.service")
node.succeed("rm -rf /var/lib/garage-metadata")
for node in nodes:
node.systemctl("start garage.service")
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should create specified buckets and keys"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
garage1.succeed("garage key list | grep testKey")
garage1.succeed("garage bucket list | grep bucket1")
garage1.succeed("garage bucket list | grep bucket2")
with subtest("should delete unspecified buckets and keys"):
garage1.succeed("garage bucket create unwantedbucket")
garage1.succeed("garage key new --name unwantedkey")
garage1.succeed("systemctl restart garage-apply.service")
garage1.fail("garage key list | grep unwantedkey")
garage1.fail("garage bucket list | grep unwantedbucket")
'';
}