From e53f766f9dce66059b4c17fad1a2843c8d2328e7 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 3 Aug 2024 02:45:19 +0200 Subject: [PATCH 01/11] cluster/services/storage: implement s3ql key format --- cluster/services/storage/garage-options.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster/services/storage/garage-options.nix b/cluster/services/storage/garage-options.nix index a423d25..a8ab84b 100644 --- a/cluster/services/storage/garage-options.nix +++ b/cluster/services/storage/garage-options.nix @@ -118,7 +118,7 @@ in }; format = mkOption { description = "Locksmith secret format."; - type = enum [ "files" "aws" "envFile" ]; + type = enum [ "files" "aws" "envFile" "s3ql" ]; default = "files"; }; owner = mkOption { @@ -291,6 +291,12 @@ in AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@ AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@ ''; + s3ql = '' + [s3c] + storage-url: s3c4:// + backend-login: @@GARAGE_KEY_ID@@ + backend-password: @@GARAGE_SECRET_KEY@@ + ''; }.${kCfg.locksmith.format}; in { ${key} = common // { -- 2.47.0 From 1a7efa6732575fe63f0eaa5f1a7bad2f65b69085 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 3 Aug 2024 02:58:20 +0200 Subject: [PATCH 02/11] modules/external-storage: support locksmith secrets --- modules/external-storage/default.nix | 18 +++++++++++++++--- modules/external-storage/filesystem-type.nix | 4 ++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/modules/external-storage/default.nix b/modules/external-storage/default.nix index 4a9d0eb..50ad622 100644 --- a/modules/external-storage/default.nix +++ b/modules/external-storage/default.nix @@ -8,6 +8,8 @@ let cfgAge = config.age; create = lib.flip lib.mapAttrs'; + + createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f; in { @@ -20,12 +22,17 @@ in fileSystems = lib.mkOption { description = "S3QL-based filesystems on top of CIFS mountpoints."; default = {}; - type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: { + type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let + authFile = if config.locksmithSecret != null then + "/run/locksmith/${config.locksmithSecret}" + else + cfgAge.secrets."storageAuth-${name}".path; + in { imports = [ ./filesystem-type.nix ]; backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}"; commonArgs = [ "--cachedir" config.cacheDir - "--authfile" cfgAge.secrets."storageAuth-${name}".path + "--authfile" authFile ] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]); })); }; @@ -57,9 +64,14 @@ in age.secrets = lib.mkMerge [ (create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; })) - (create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; })) + (createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; })) ]; + services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: { + name = fs.unitName; + value = [ fs.locksmithSecret ]; + }); + fileSystems = create cfg.underlays (name: ul: { name = ul.mountpoint; value = { diff --git a/modules/external-storage/filesystem-type.nix b/modules/external-storage/filesystem-type.nix index 8cece5d..6f37094 100644 --- a/modules/external-storage/filesystem-type.nix +++ b/modules/external-storage/filesystem-type.nix @@ -22,6 +22,10 @@ with lib; authFile = mkOption { type = types.path; }; + locksmithSecret = mkOption { + type = with types; nullOr str; + default = null; + }; cacheDir = mkOption { type = types.path; default = "/var/cache/remote-storage/${name}"; -- 2.47.0 From 59ff96697d78aa7bb977443fa31328fe880cd7bd Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 16 Aug 2024 21:57:24 +0200 Subject: [PATCH 03/11] cluster/services/storage: use incandescence --- cluster/services/storage/default.nix | 1 + cluster/services/storage/garage-options.nix | 170 ++++++++------------ cluster/services/storage/incandescence.nix | 10 ++ 3 files changed, 79 insertions(+), 102 deletions(-) create mode 100644 cluster/services/storage/incandescence.nix diff --git a/cluster/services/storage/default.nix b/cluster/services/storage/default.nix index d69aeec..e4da92c 100644 --- a/cluster/services/storage/default.nix +++ b/cluster/services/storage/default.nix @@ -7,6 +7,7 @@ in { imports = [ ./options.nix + ./incandescence.nix ]; services.storage = { diff --git a/cluster/services/storage/garage-options.nix b/cluster/services/storage/garage-options.nix index a8ab84b..64eaacd 100644 --- a/cluster/services/storage/garage-options.nix +++ b/cluster/services/storage/garage-options.nix @@ -26,62 +26,6 @@ let sleep 1 done } - # FIXME: returns bogus empty string when one of the lists is empty - diffAdded() { - comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort) - } - diffRemoved() { - comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort) - } - # FIXME: this does not handle list items with spaces - listKeys() { - garage key list | tail -n +2 | grep -ow '[^ ]*$' || true - } - ensureKeys() { - old="$(listKeys)" - if [[ -z "$1" ]]; then - for key in $old; do - garage key delete --yes "$key" - done - elif [[ -z "$old" ]]; then - for key in $1; do - # don't print secret key - garage key new --name "$key" >/dev/null - echo Key "$key" was created. - done - else - diffAdded "$old" "$1" | while read key; do - # don't print secret key - garage key new --name "$key" >/dev/null - echo Key "$key" was created. - done - diffRemoved "$old" "$1" | while read key; do - garage key delete --yes "$key" - done - fi - } - listBuckets() { - garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true - } - ensureBuckets() { - old="$(listBuckets)" - if [[ -z "$1" ]]; then - for bucket in $old; do - garage bucket delete --yes "$bucket" - done - elif [[ -z "$old" ]]; then - for bucket in $1; do - garage bucket create "$bucket" - done - else - diffAdded "$old" "$1" | while read bucket; do - garage bucket create "$bucket" - done - diffRemoved "$old" "$1" | while read bucket; do - garage bucket delete --yes "$bucket" - done - fi - } ''; in @@ -203,9 +147,7 @@ in garage layout apply --version 1 ''; }; - garage-apply = { - distributed.enable = true; - wantedBy = [ "garage.service" "multi-user.target" ]; + garage-ready = { wants = [ "garage.service" ]; after = [ "garage.service" "garage-layout-init.service" ]; path = [ config.services.garage.package ]; @@ -219,54 +161,78 @@ in script = '' source ${garageShellLibrary} waitForGarageOperational - - ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}' - ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}' - - # key permissions - ${lib.pipe cfg.keys [ - (lib.mapAttrsToList (key: kCfg: '' - garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null - '')) - (lib.concatStringsSep "\n") - ]} - - # bucket permissions - ${lib.pipe cfg.buckets [ - (lib.mapAttrsToList (bucket: bCfg: - lib.mapAttrsToList (key: perms: '' - garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)} - garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))} - '') bCfg.allow - )) - lib.flatten - (lib.concatStringsSep "\n") - ]} - - # bucket quotas - ${lib.pipe cfg.buckets [ - (lib.mapAttrsToList (bucket: bCfg: '' - garage bucket set-quotas '${bucket}' \ - --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \ - --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}' - '')) - (lib.concatStringsSep "\n") - ]} - - # bucket website access - ${lib.pipe cfg.buckets [ - (lib.mapAttrsToList (bucket: bCfg: '' - garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}' - '')) - (lib.concatStringsSep "\n") - ]} ''; }; }; + services.incandescence.providers.garage = { + locksmith = true; + wantedBy = [ "garage.service" "multi-user.target" ]; + partOf = [ "garage.service" ]; + wants = [ "garage-ready.service" ]; + after = [ "garage-ready.service" ]; + + packages = [ + config.services.garage.package + ]; + formulae = { + key = { + destroyAfterDays = 0; + create = key: '' + if [[ "$(garage key info ${lib.escapeShellArg key} 2>&1 >/dev/null)" == "Error: 0 matching keys" ]]; then + # don't print secret key + garage key new --name ${lib.escapeShellArg key} >/dev/null + echo Key ${lib.escapeShellArg key} was created. + else + echo "Key already exists, assuming ownership" + fi + ''; + destroy = '' + garage key delete --yes "$OBJECT" + ''; + change = key: let + kCfg = cfg.keys.${key}; + in '' + garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null + ''; + }; + bucket = { + deps = [ "key" ]; + destroyAfterDays = 30; + create = bucket: '' + if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == "Error: Bucket not found" ]]; then + garage bucket create ${lib.escapeShellArg bucket} + else + echo "Bucket already exists, assuming ownership" + fi + ''; + destroy = '' + garage bucket delete --yes "$OBJECT" + ''; + change = bucket: let + bCfg = cfg.buckets.${bucket}; + in '' + # permissions + ${lib.concatStringsSep "\n" (lib.flatten ( + lib.mapAttrsToList (key: perms: '' + garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)} + garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))} + '') bCfg.allow + ))} + + # quotas + garage bucket set-quotas ${lib.escapeShellArg bucket} \ + --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \ + --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}' + + # website access + garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket} + ''; + }; + }; + }; + services.locksmith.providers.garage = { - wantedBy = [ "garage-apply.service" ]; - after = [ "garage-apply.service" ]; secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let common = { inherit (kCfg.locksmith) mode owner group nodes; diff --git a/cluster/services/storage/incandescence.nix b/cluster/services/storage/incandescence.nix new file mode 100644 index 0000000..a798905 --- /dev/null +++ b/cluster/services/storage/incandescence.nix @@ -0,0 +1,10 @@ +{ config, lib, ... }: + +{ + incandescence.providers.garage = { + objects = { + key = lib.attrNames config.garage.keys; + bucket = lib.attrNames config.garage.buckets; + }; + }; +} -- 2.47.0 From 7287fcb5db9f1eb32f8eef689661a448d451b3dc Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 23 Jul 2024 02:47:13 +0200 Subject: [PATCH 04/11] cluster/services/storage: test in simulacrum --- cluster/services/storage/default.nix | 8 ++ cluster/services/storage/garage.nix | 2 + .../simulacrum/snakeoil-rpc-secret.nix | 3 + .../services/storage/simulacrum/test-data.nix | 8 ++ cluster/services/storage/simulacrum/test.nix | 105 ++++++++++++++++++ 5 files changed, 126 insertions(+) create mode 100644 cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix create mode 100644 cluster/services/storage/simulacrum/test-data.nix create mode 100644 cluster/services/storage/simulacrum/test.nix diff --git a/cluster/services/storage/default.nix b/cluster/services/storage/default.nix index e4da92c..4ab6ac0 100644 --- a/cluster/services/storage/default.nix +++ b/cluster/services/storage/default.nix @@ -8,6 +8,7 @@ in imports = [ ./options.nix ./incandescence.nix + ./simulacrum/test-data.nix ]; services.storage = { @@ -36,6 +37,8 @@ in ./garage.nix ./garage-options.nix ./garage-layout.nix + ] ++ lib.optionals config.simulacrum [ + ./simulacrum/snakeoil-rpc-secret.nix ]; garageConfig = [ ./garage-gateway.nix @@ -49,6 +52,11 @@ in garageInternal = [ ./garage-internal.nix ]; garageExternal = [ ./garage-external.nix ]; }; + simulacrum = { + enable = true; + deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ]; + settings = ./simulacrum/test.nix; + }; }; links = { diff --git a/cluster/services/storage/garage.nix b/cluster/services/storage/garage.nix index a319ba1..93d0bcf 100644 --- a/cluster/services/storage/garage.nix +++ b/cluster/services/storage/garage.nix @@ -63,6 +63,8 @@ in }; systemd.services.garage = { + requires = [ "consul-ready.service" ]; + after = [ "consul-ready.service" ]; unitConfig = { RequiresMountsFor = [ cfg.settings.data_dir ]; }; diff --git a/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix b/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix new file mode 100644 index 0000000..501bf1b --- /dev/null +++ b/cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix @@ -0,0 +1,3 @@ +{ + environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +} diff --git a/cluster/services/storage/simulacrum/test-data.nix b/cluster/services/storage/simulacrum/test-data.nix new file mode 100644 index 0000000..c688365 --- /dev/null +++ b/cluster/services/storage/simulacrum/test-data.nix @@ -0,0 +1,8 @@ +{ config, lib, ... }: + +{ + garage = lib.mkIf config.simulacrum { + keys.testkey = {}; + buckets.testbucket.allow.testKey = [ "read" "write" ]; + }; +} diff --git a/cluster/services/storage/simulacrum/test.nix b/cluster/services/storage/simulacrum/test.nix new file mode 100644 index 0000000..a1fcf17 --- /dev/null +++ b/cluster/services/storage/simulacrum/test.nix @@ -0,0 +1,105 @@ +{ cluster, lib, ... }: + +let + inherit (cluster.config.services.storage) nodes; + + firstGarageNode = lib.elemAt nodes.garage 0; +in + +{ + nodes = lib.genAttrs nodes.garage (node: { + services.garage = { + layout.initial = lib.genAttrs nodes.garage (_: { + capacity = lib.mkOverride 51 1000; + }); + }; + specialisation.modifiedLayout = { + inheritParentConfig = true; + configuration = { + services.garage = { + layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000; + }; + system.ascensions.garage-layout.incantations = lib.mkForce (i: [ + (i.runGarage '' + garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)" + garage layout apply --version 2 + '') + ]); + }; + }; + }); + + testScript = '' + import json + nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')] + garage1 = nodes[0] + + start_all() + + with subtest("should bootstrap new cluster"): + for node in nodes: + node.wait_for_unit("garage.service") + + for node in nodes: + node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") + + with subtest("should apply new layout with ascension"): + for node in nodes: + node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1') + + for node in nodes: + node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test") + + for node in nodes: + node.wait_until_succeeds("garage layout show | grep -w 2000") + assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") + assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l") + + consulConfig = json.loads(garage1.succeed("cat /etc/consul.json")) + addr = consulConfig["addresses"]["http"] + port = consulConfig["ports"]["http"] + setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}" + with subtest("should apply new layout from scratch"): + for node in nodes: + node.systemctl("stop garage.service") + node.succeed("rm -rf /var/lib/garage-metadata") + garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage") + + for node in nodes: + node.systemctl("start garage.service") + + for node in nodes: + node.wait_for_unit("garage.service") + + for node in nodes: + node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") + + for node in nodes: + node.wait_until_succeeds("garage layout show | grep -w 2000") + assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") + assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l") + + with subtest("should create specified buckets and keys"): + for node in nodes: + node.wait_for_unit("incandescence-garage.target") + garage1.succeed("garage key list | grep testkey") + garage1.succeed("garage bucket list | grep testbucket") + + with subtest("should delete unspecified keys"): + garage1.succeed("garage bucket create unwantedbucket") + garage1.succeed("garage key new --name unwantedkey") + garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true") + garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true") + garage1.succeed("systemctl restart garage.service") + garage1.wait_for_unit("incandescence-garage.target") + garage1.fail("garage key list | grep unwantedkey") + garage1.succeed("garage bucket list | grep unwantedbucket") + + with subtest("should delete unspecified buckets after grace period"): + garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1") + garage1.succeed("systemctl restart garage.service") + garage1.wait_for_unit("incandescence-garage.target") + garage1.fail("garage bucket list | grep unwantedbucket") + ''; +} + -- 2.47.0 From f3039ec4027a775aa8ee2cfa0acc114f6dbe0421 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 3 Aug 2024 01:33:22 +0200 Subject: [PATCH 05/11] checks/garage: drop --- packages/checks/default.nix | 6 -- packages/checks/garage.nix | 155 ------------------------------------ 2 files changed, 161 deletions(-) delete mode 100644 packages/checks/garage.nix diff --git a/packages/checks/default.nix b/packages/checks/default.nix index ed21da1..9f3d375 100644 --- a/packages/checks/default.nix +++ b/packages/checks/default.nix @@ -15,12 +15,6 @@ in inherit (config) cluster; }; - garage = pkgs.callPackage ./garage.nix { - inherit (self'.packages) garage consul; - inherit (self) nixosModules; - inherit (config) cluster; - }; - ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix { inherit (self) nixosModules; previous = timeMachine.preUnstable; diff --git a/packages/checks/garage.nix b/packages/checks/garage.nix deleted file mode 100644 index 1dc759e..0000000 --- a/packages/checks/garage.nix +++ /dev/null @@ -1,155 +0,0 @@ -{ testers, nixosModules, cluster, garage, consul }: - -testers.runNixOSTest { - name = "garage"; - - imports = [ - ./modules/consul.nix - ]; - - extraBaseModules.services.consul.package = consul; - - nodes = let - common = { config, lib, ... }: let - inherit (config.networking) hostName primaryIPAddress; - in { - imports = lib.flatten [ - ./modules/nixos/age-dummy-secrets - ./modules/nixos/age-dummy-secrets/options.nix - nixosModules.ascensions - nixosModules.systemd-extras - nixosModules.consul-distributed-services - nixosModules.port-magic - cluster.config.services.storage.nixos.garage - cluster.config.services.storage.nixos.garageInternal - cluster.config.services.consul.nixos.ready - ]; - options.services.locksmith.providers = lib.mkOption { - type = lib.types.raw; - }; - config = { - links.consulAgent = { - protocol = "http"; - hostname = "consul"; - port = 8500; - }; - _module.args = { - depot.packages = { inherit garage; }; - cluster.config = { - hostLinks.${hostName} = { - garageRpc.tuple = "${primaryIPAddress}:3901"; - garageS3.tuple = "${primaryIPAddress}:8080"; - garageWeb.tuple = "${primaryIPAddress}:8081"; - }; - links.garageWeb.hostname = "web.garage.example.com"; - vars.meshNet.cidr = "192.168.0.0/16"; - }; - }; - environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - networking.firewall.allowedTCPPorts = [ 3901 8080 ]; - services.garage = { - layout.initial = lib.mkOverride 51 { - garage1 = { zone = "dc1"; capacity = 1000; }; - garage2 = { zone = "dc1"; capacity = 1000; }; - garage3 = { zone = "dc1"; capacity = 1000; }; - }; - }; - system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]); - specialisation.modifiedLayout = { - inheritParentConfig = true; - configuration = { - services.garage = { - layout.initial = lib.mkForce { - garage1 = { zone = "dc1"; capacity = 2000; }; - garage2 = { zone = "dc1"; capacity = 1000; }; - garage3 = { zone = "dc1"; capacity = 1000; }; - }; - keys.testKey.allow.createBucket = true; - buckets = { - bucket1 = { - allow.testKey = [ "read" "write" ]; - quotas = { - maxObjects = 300; - maxSize = 400 * 1024 * 1024; - }; - }; - bucket2 = { - allow.testKey = [ "read" ]; - }; - }; - }; - system.ascensions.garage-layout.incantations = lib.mkForce (i: [ - (i.runGarage '' - garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)" - garage layout apply --version 2 - '') - ]); - }; - }; - }; - }; - in { - garage1.imports = [ common ]; - garage2.imports = [ common ]; - garage3.imports = [ common ]; - }; - - testScript = { nodes, ... }: /*python*/ '' - nodes = [garage1, garage2, garage3] - - start_all() - - with subtest("should bootstrap new cluster"): - for node in nodes: - node.wait_for_unit("garage.service") - - for node in nodes: - node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") - - with subtest("should apply new layout with ascension"): - for node in nodes: - node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1') - - for node in nodes: - node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test") - - for node in nodes: - node.wait_until_succeeds("garage layout show | grep -w 2000") - assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") - assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l") - - with subtest("should apply new layout from scratch"): - for node in nodes: - node.systemctl("stop garage.service") - node.succeed("rm -rf /var/lib/garage-metadata") - - for node in nodes: - node.systemctl("start garage.service") - - for node in nodes: - node.wait_for_unit("garage.service") - - for node in nodes: - node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") - - for node in nodes: - node.wait_until_succeeds("garage layout show | grep -w 2000") - assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") - assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l") - - with subtest("should create specified buckets and keys"): - for node in nodes: - node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating') - garage1.succeed("garage key list | grep testKey") - garage1.succeed("garage bucket list | grep bucket1") - garage1.succeed("garage bucket list | grep bucket2") - - with subtest("should delete unspecified buckets and keys"): - garage1.succeed("garage bucket create unwantedbucket") - garage1.succeed("garage key new --name unwantedkey") - garage1.succeed("systemctl restart garage-apply.service") - - garage1.fail("garage key list | grep unwantedkey") - garage1.fail("garage bucket list | grep unwantedbucket") - ''; -} -- 2.47.0 From 46f04058f9ea43022434adf8470548a324540b89 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 4 Aug 2024 23:45:29 +0200 Subject: [PATCH 06/11] cluster/services/storage: use locksmith secrets for external storage --- cluster/services/storage/default.nix | 5 ++++- cluster/services/storage/external.nix | 2 +- .../storage/secrets/external-storage-auth-prophet.age | 11 ----------- 3 files changed, 5 insertions(+), 13 deletions(-) delete mode 100644 cluster/services/storage/secrets/external-storage-auth-prophet.age diff --git a/cluster/services/storage/default.nix b/cluster/services/storage/default.nix index 4ab6ac0..a4c5b72 100644 --- a/cluster/services/storage/default.nix +++ b/cluster/services/storage/default.nix @@ -95,7 +95,10 @@ in }; garage = { - keys.storage-prophet = {}; + keys.storage-prophet.locksmith = { + nodes = [ "prophet" ]; + format = "s3ql"; + }; buckets.storage-prophet = { allow.storage-prophet = [ "read" "write" ]; }; diff --git a/cluster/services/storage/external.nix b/cluster/services/storage/external.nix index 971d9a6..d1514e5 100644 --- a/cluster/services/storage/external.nix +++ b/cluster/services/storage/external.nix @@ -8,7 +8,7 @@ in services.external-storage = { fileSystems.external = { mountpoint = "/srv/storage"; - authFile = ./secrets/external-storage-auth-${hostName}.age; + locksmithSecret = "garage-storage-${hostName}"; backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}"; backendOptions = [ "disable-expect100" ]; }; diff --git a/cluster/services/storage/secrets/external-storage-auth-prophet.age b/cluster/services/storage/secrets/external-storage-auth-prophet.age deleted file mode 100644 index bad24c8..0000000 --- a/cluster/services/storage/secrets/external-storage-auth-prophet.age +++ /dev/null @@ -1,11 +0,0 @@ -age-encryption.org/v1 --> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w -2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA --> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c -0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ --> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w -LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4 ---- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w -gN 8\K!p 7k#u*{}T0|@ E>z'-RxKzBn*0~OV4q]^(>-3e0a.oC)4g7NzɔnMx6'[6w?i=vEJB -9gi"Q -ٮ \ No newline at end of file -- 2.47.0 From 9272c555bca226c97c4d8d535bb980cdf1088ec0 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 9 Aug 2024 23:46:11 +0200 Subject: [PATCH 07/11] modules/external-storage: implement detectFs for s3c4 --- modules/external-storage/default.nix | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/modules/external-storage/default.nix b/modules/external-storage/default.nix index 50ad622..b4ddd12 100644 --- a/modules/external-storage/default.nix +++ b/modules/external-storage/default.nix @@ -109,7 +109,13 @@ in value = let isUnderlay = fs.underlay != null; - fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend); + backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend; + + fsType = if isUnderlay then "local" else lib.head backendParts; + + s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1; + + s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2; localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend); in { @@ -132,8 +138,12 @@ in ExecStartPre = map lib.escapeShellArgs [ [ (let + authFile = if fs.locksmithSecret != null then + "/run/locksmith/${fs.locksmithSecret}" + else + cfgAge.secrets."storageAuth-${name}".path; mkfsEncrypted = '' - ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \ + ${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \ | cut -d' ' -f2- \ | ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}' ''; @@ -144,6 +154,11 @@ in detectFs = { local = "test -e ${localBackendPath}/s3ql_metadata"; + s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" '' + export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)" + export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)" + ${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null + ''; }.${fsType} or null; in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) '' if ! ${detectFs}; then -- 2.47.0 From ad65ad500e19f4ad3d87f5d767ca047a9a1cd495 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 11 Aug 2024 01:29:19 +0200 Subject: [PATCH 08/11] cluster/services/storage: define snakeoil passphrase for heresy, ensure encryption --- cluster/services/storage/default.nix | 2 ++ cluster/services/storage/heresy.nix | 1 + .../storage/simulacrum/snakeoil-heresy-passphrase.nix | 8 ++++++++ 3 files changed, 11 insertions(+) create mode 100644 cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix diff --git a/cluster/services/storage/default.nix b/cluster/services/storage/default.nix index a4c5b72..bc5de89 100644 --- a/cluster/services/storage/default.nix +++ b/cluster/services/storage/default.nix @@ -32,6 +32,8 @@ in heresy = [ ./heresy.nix ./s3ql-upgrades.nix + ] ++ lib.optionals config.simulacrum [ + ./simulacrum/snakeoil-heresy-passphrase.nix ]; garage = [ ./garage.nix diff --git a/cluster/services/storage/heresy.nix b/cluster/services/storage/heresy.nix index ace3343..ed428d7 100644 --- a/cluster/services/storage/heresy.nix +++ b/cluster/services/storage/heresy.nix @@ -11,6 +11,7 @@ unitDescription = "Heresy Filesystem"; authFile = ./secrets/heresy-encryption-key.age; underlay = "heresy"; + encrypt = true; }; }; } diff --git a/cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix b/cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix new file mode 100644 index 0000000..bcfb410 --- /dev/null +++ b/cluster/services/storage/simulacrum/snakeoil-heresy-passphrase.nix @@ -0,0 +1,8 @@ +{ + environment.etc."dummy-secrets/storageAuth-heresy".text = '' + [local] + storage-url: local:// + fs-passphrase: simulacrum + ''; +} + -- 2.47.0 From dfec17da629cffb6160bdc11737702acf178ad4b Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 16 Aug 2024 22:12:30 +0200 Subject: [PATCH 09/11] checks/s3ql-upgrade: stub locksmith options --- packages/checks/s3ql-upgrade.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/checks/s3ql-upgrade.nix b/packages/checks/s3ql-upgrade.nix index c422e83..4a6d5fc 100644 --- a/packages/checks/s3ql-upgrade.nix +++ b/packages/checks/s3ql-upgrade.nix @@ -10,6 +10,9 @@ testers.runNixOSTest { nixosModules.systemd-extras ./modules/nixos/age-dummy-secrets ./modules/nixos/age-dummy-secrets/options.nix + { + options.services.locksmith = lib.mkSinkUndeclaredOptions { }; + } ]; _module.args.depot.packages = { inherit (previous.packages.${system}) s3ql; }; -- 2.47.0 From 81fafdfd04cd97477462c5f9727d08f40c136644 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 16 Aug 2024 22:28:03 +0200 Subject: [PATCH 10/11] cluster/services/wireguard: stub locksmith options in test --- cluster/services/wireguard/test.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/services/wireguard/test.nix b/cluster/services/wireguard/test.nix index dfa7e92..7f9bf3a 100644 --- a/cluster/services/wireguard/test.nix +++ b/cluster/services/wireguard/test.nix @@ -1,6 +1,8 @@ { cluster, lib, ... }: { + defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { }; + testScript = '' start_all() ${lib.pipe cluster.config.services.wireguard.nodes.mesh [ -- 2.47.0 From 51cf6dabc20ec595c990aca023cd2bc6f17815df Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 16 Aug 2024 22:28:42 +0200 Subject: [PATCH 11/11] cluster/services/consul: stub locksmith options in test --- cluster/services/consul/test.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster/services/consul/test.nix b/cluster/services/consul/test.nix index f4aa2eb..d3ed32d 100644 --- a/cluster/services/consul/test.nix +++ b/cluster/services/consul/test.nix @@ -1,4 +1,8 @@ +{ lib, ... }: + { + defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { }; + testScript = '' import json -- 2.47.0