diff --git a/cluster/services/storage/default.nix b/cluster/services/storage/default.nix index 531a14b..8f19915 100644 --- a/cluster/services/storage/default.nix +++ b/cluster/services/storage/default.nix @@ -1,12 +1,38 @@ +{ config, lib, ... }: + +let + meshIpForNode = name: config.vars.mesh.${name}.meshIp; +in + { services.storage = { nodes = { external = [ "prophet" ]; heresy = [ "VEGAS" ]; + garage = [ "checkmate" "prophet" "VEGAS" ]; + garageInternal = [ "VEGAS" ]; + garageExternal = [ "checkmate" "prophet" ]; }; nixos = { external = [ ./external.nix ]; heresy = [ ./heresy.nix ]; + garage = [ + ./garage.nix + ./garage-options.nix + ./garage-layout.nix + ]; + garageInternal = [ ./garage-internal.nix ]; + garageExternal = [ ./garage-external.nix ]; }; }; + + hostLinks = lib.genAttrs config.services.storage.nodes.garage (name: { + garageRpc = { + ipv4 = meshIpForNode name; + }; + garageS3 = { + protocol = "http"; + ipv4 = meshIpForNode name; + }; + }); } diff --git a/cluster/services/storage/garage-external.nix b/cluster/services/storage/garage-external.nix new file mode 100644 index 0000000..01940d0 --- /dev/null +++ b/cluster/services/storage/garage-external.nix @@ -0,0 +1,15 @@ +{ config, ... }: + +{ + services.external-storage = { + underlays.garage = { + subUser = "sub1"; + credentialsFile = ./secrets/storage-box-credentials.age; + path = "/garage/${config.networking.hostName}"; + inherit (config.users.users.garage) uid; + inherit (config.users.groups.garage) gid; + }; + }; + + services.garage.settings.data_dir = config.services.external-storage.underlays.garage.mountpoint; +} diff --git a/cluster/services/storage/garage-internal.nix b/cluster/services/storage/garage-internal.nix new file mode 100644 index 0000000..bfe8d47 --- /dev/null +++ b/cluster/services/storage/garage-internal.nix @@ -0,0 +1,11 @@ +let + dataDir = "/srv/storage/private/garage"; +in + +{ + systemd.tmpfiles.rules = [ + "d '${dataDir}' 0700 garage garage -" + ]; + + services.garage.settings.data_dir = dataDir; +} diff --git a/cluster/services/storage/garage-layout.nix b/cluster/services/storage/garage-layout.nix new file mode 100644 index 0000000..4e707e1 --- /dev/null +++ b/cluster/services/storage/garage-layout.nix @@ -0,0 +1,14 @@ +{ + system.ascensions.garage-layout = { + distributed = true; + requiredBy = [ "garage.service" ]; + after = [ "garage.service" "garage-layout-init.service" ]; + incantations = i: [ ]; + }; + + services.garage.layout.initial = { + checkmate = { zone = "eu-central"; capacity = 1000; }; + prophet = { zone = "eu-central"; capacity = 1000; }; + VEGAS = { zone = "eu-central"; capacity = 1000; }; + }; +} diff --git a/cluster/services/storage/garage-options.nix b/cluster/services/storage/garage-options.nix new file mode 100644 index 0000000..c5e6509 --- /dev/null +++ b/cluster/services/storage/garage-options.nix @@ -0,0 +1,232 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.garage; + + garageShellLibrary = pkgs.writeText "garage-shell-library.sh" '' + getNodeId() { + nodeId="" + while [[ -z "$nodeId" ]]; do + nodeId="$(garage status | grep 'NO ROLE ASSIGNED' | grep -wm1 "$1" | cut -d' ' -f1)" + if [[ $? -ne 0 ]]; then + echo "Waiting for node $1 to appear..." 2>/dev/null + sleep 1 + fi + done + echo "$nodeId" + } + waitForGarage() { + while ! garage status >/dev/null 2>/dev/null; do + sleep 1 + done + } + waitForGarageOperational() { + waitForGarage + while garage layout show | grep -qwm1 '^Current cluster layout version: 0'; do + sleep 1 + done + } + # FIXME: returns bogus empty string when one of the lists is empty + diffAdded() { + comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort) + } + diffRemoved() { + comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort) + } + # FIXME: this does not handle list items with spaces + listKeys() { + garage key list | tail -n +2 | grep -ow '[^ ]*$' || true + } + ensureKeys() { + old="$(listKeys)" + if [[ -z "$1" ]]; then + for key in $old; do + garage key delete --yes "$key" + done + elif [[ -z "$old" ]]; then + for key in $1; do + # don't print secret key + garage key new --name "$key" >/dev/null + echo Key "$key" was created. + done + else + diffAdded "$old" "$1" | while read key; do + # don't print secret key + garage key new --name "$key" >/dev/null + echo Key "$key" was created. + done + diffRemoved "$old" "$1" | while read key; do + garage key delete --yes "$key" + done + fi + } + listBuckets() { + garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true + } + ensureBuckets() { + old="$(listBuckets)" + if [[ -z "$1" ]]; then + for bucket in $old; do + garage bucket delete --yes "$bucket" + done + elif [[ -z "$old" ]]; then + for bucket in $1; do + garage bucket create "$bucket" + done + else + diffAdded "$old" "$1" | while read bucket; do + garage bucket create "$bucket" + done + diffRemoved "$old" "$1" | while read bucket; do + garage bucket delete --yes "$bucket" + done + fi + } + ''; +in + +{ + options.services.garage = with lib; { + layout.initial = mkOption { + default = {}; + type = with types; attrsOf (submodule { + options = { + zone = mkOption { + type = types.str; + }; + capacity = mkOption { + type = types.ints.positive; + }; + }; + }); + }; + keys = mkOption { + type = with types; attrsOf (submodule { + options.allow = { + createBucket = mkOption { + description = "Allow the key to create new buckets."; + type = bool; + default = false; + }; + }; + }); + default = {}; + }; + buckets = mkOption { + type = with types; attrsOf (submodule { + options = { + allow = mkOption { + description = "List of permissions to grant on this bucket, grouped by key name."; + type = attrsOf (listOf (enum [ "read" "write" "owner" ])); + default = {}; + }; + quotas = { + maxObjects = mkOption { + description = "Maximum number of objects in this bucket. Null for unlimited."; + type = nullOr ints.positive; + default = null; + }; + maxSize = mkOption { + description = "Maximum size of this bucket in bytes. Null for unlimited."; + type = nullOr ints.positive; + default = null; + }; + }; + }; + }); + default = {}; + }; + }; + + config = { + system.extraIncantations = { + runGarage = i: script: i.execShellWith [ config.services.garage.package ] '' + source ${garageShellLibrary} + waitForGarage + ${script} + ''; + }; + + systemd.services = { + garage-layout-init = { + distributed.enable = true; + wantedBy = [ "garage.service" "multi-user.target" ]; + wants = [ "garage.service" ]; + after = [ "garage.service" ]; + path = [ config.services.garage.package ]; + + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = "1800s"; + Restart = "on-failure"; + RestartSec = "10s"; + }; + script = '' + source ${garageShellLibrary} + waitForGarage + + if [[ "$(garage layout show | grep -m1 '^Current cluster layout version:' | cut -d: -f2 | tr -d ' ')" != "0" ]]; then + exit 0 + fi + + ${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: layout: '' + garage layout assign -z '${layout.zone}' -c '${toString layout.capacity}' "$(getNodeId '${name}')" + '') cfg.layout.initial)} + + garage layout apply --version 1 + ''; + }; + garage-apply = { + distributed.enable = true; + wantedBy = [ "garage.service" "multi-user.target" ]; + wants = [ "garage.service" ]; + after = [ "garage.service" "garage-layout-init.service" ]; + path = [ config.services.garage.package ]; + + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = "1800s"; + Restart = "on-failure"; + RestartSec = "10s"; + }; + script = '' + source ${garageShellLibrary} + waitForGarageOperational + + ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}' + ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}' + + # key permissions + ${lib.pipe cfg.keys [ + (lib.mapAttrsToList (key: kCfg: '' + garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null + '')) + (lib.concatStringsSep "\n") + ]} + + # bucket permissions + ${lib.pipe cfg.buckets [ + (lib.mapAttrsToList (bucket: bCfg: + lib.mapAttrsToList (key: perms: '' + garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)} + garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))} + '') bCfg.allow + )) + lib.flatten + (lib.concatStringsSep "\n") + ]} + + # bucket quotas + ${lib.pipe cfg.buckets [ + (lib.mapAttrsToList (bucket: bCfg: '' + garage bucket set-quotas '${bucket}' \ + --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \ + --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}' + '')) + (lib.concatStringsSep "\n") + ]} + ''; + }; + }; + }; +} diff --git a/cluster/services/storage/garage.nix b/cluster/services/storage/garage.nix new file mode 100644 index 0000000..ce80008 --- /dev/null +++ b/cluster/services/storage/garage.nix @@ -0,0 +1,71 @@ +{ cluster, config, depot, lib, ... }: + +let + inherit (cluster.config) vars; + inherit (config.networking) hostName; + + links = cluster.config.hostLinks.${hostName}; + + cfg = config.services.garage; + + # fixed uid so we know how to mount the underlay + uid = 777; + gid = 777; +in + +{ + age.secrets.garageRpcSecret = { + file = ./secrets/garage-rpc-secret.age; + owner = "garage"; + group = "garage"; + }; + + services.garage = { + enable = true; + package = depot.packages.garage; + settings = { + replication_mode = 3; + block_size = 16 * 1024 * 1024; + db_engine = "lmdb"; + metadata_dir = "/var/lib/garage-metadata"; + rpc_bind_addr = links.garageRpc.tuple; + rpc_public_addr = links.garageRpc.tuple; + rpc_secret_file = config.age.secrets.garageRpcSecret.path; + consul_discovery = { + consul_http_addr = "http://127.0.0.1:8500"; + api = "agent"; + service_name = "garage-discovery"; + }; + s3_api = { + api_bind_addr = links.garageS3.tuple; + s3_region = "us-east-1"; + }; + }; + }; + + users = { + users.garage = { + inherit uid; + group = "garage"; + }; + groups.garage = { + inherit gid; + }; + }; + + systemd.services.garage = { + unitConfig = { + RequiresMountsFor = [ cfg.settings.data_dir ]; + }; + serviceConfig = { + IPAddressDeny = [ "any" ]; + IPAddressAllow = [ "127.0.0.1/8" vars.meshNet.cidr ]; + DynamicUser = false; + PrivateTmp = true; + ProtectSystem = true; + User = "garage"; + Group = "garage"; + StateDirectory = lib.removePrefix "/var/lib/" cfg.settings.metadata_dir; + }; + }; +} diff --git a/cluster/services/storage/secrets/garage-rpc-secret.age b/cluster/services/storage/secrets/garage-rpc-secret.age new file mode 100644 index 0000000..1698355 --- /dev/null +++ b/cluster/services/storage/secrets/garage-rpc-secret.age @@ -0,0 +1,17 @@ +age-encryption.org/v1 +-> ssh-ed25519 NO562A WymFH+09McLeT2+o487RiKzExeG7QKTwwuERimfVygs +dLTYSgo35f3cvhRWzhnutjSSRChBhhdX+87Yd6H6A5w +-> ssh-ed25519 5/zT0w izNXf7mqdiDyc6AN6kbQfMHEtGxiI6+DnyiVxaSqFDo +mXu7h0oY4UgiMojcNKedWXwvnQJmq4wyFkE1ZVTlXSk +-> ssh-ed25519 TCgorQ 7ueU0wcT7A5bI6/JVKuIAj/jhh7TuVEfDCullS/Ws0c +nri4g0jk8hdGMYgxjarklPBmzEBfXv2tgys8xPJ0UVI +-> ssh-ed25519 d3WGuA 3LNotYdeRf4QP7bEdjx9NHInFVgRCFq91xjixu96mBI +Gj0LYo6dTVIcSOXTtqdug+zX0l6UIigBu5obEgIMK00 +-> ssh-ed25519 YIaSKQ DkCXHGHVP92yEBKlELG/mWPkbq4lXoAmFVXmemC6Sxw +YE4kSaNFVcS3txw94o5WTOKXe6xYbQALyzxBWDlZsCA +-> ;",X-grease &l-# ,k +FUMCq19iC2wN9U5NBYrHmCdUKB+p5AOnYGiA3LeWTJK3f899Bht0ZbSl7kL1l4w8 +4+n1ZbUITvPNOg +--- Fri66T6QtwEkou+ThWbOy0m651htF4yedOH6tL0VX0E +Ѹ$\%e].I+GӘ +A/k8iT#{4@h8 I \MgKGxaqO \ No newline at end of file diff --git a/cluster/services/storage/secrets/storage-box-credentials.age b/cluster/services/storage/secrets/storage-box-credentials.age index efed247..2375bad 100644 --- a/cluster/services/storage/secrets/storage-box-credentials.age +++ b/cluster/services/storage/secrets/storage-box-credentials.age @@ -1,13 +1,17 @@ age-encryption.org/v1 --> ssh-ed25519 NO562A zJvkOwBBlON92XKax+VTvXyqGFajgwzt091jNrRzjjI -1a8/gXx4DQu03Rewebxdz5BpEi3zI7DXTOcY2OLIUck --> ssh-ed25519 5/zT0w cR4KCvOxvEV5KhEL+4HEcwKv5dKC4dT+yVok8P6Y/1c -UCPh+WJ9opCFkDZgGBzyep8xZ58P6jLP1thMOpsx5rQ --> ssh-ed25519 d3WGuA nwgv+GeEw1Yc6gYCN39zfDyFg/NGxazU9tMkykmwhBc -b+1EyRRSCvUWdMKcsEZA5BAnqQHO703Bwp5xtLPYok4 --> ssh-ed25519 YIaSKQ FtSyO68mVkMEbMZV9w/H4tj7ms7gKOT+jB9n5MG5NAE -B8Id7KRU9a19oWfmDcCN4zUb04AMZ4Z+AETyCvxeC7c --> zAL;6ee-grease O]^kl 0`0G tf+sVH= `+K,h=HL -ZO4H8I1upQszxI6GBalw1hUhSAhRRhhePbE ---- lfbOCcNCO7rc1os2qEc9Bj0s6+bxJujVzmdL3lWdm3g -em1uUV#jTqe=2o9y23?jT1GBA_ü3g1-rL)˖gR  \ No newline at end of file +-> ssh-ed25519 NO562A wZnh+3ROEclIIvMM4EhbUqytTwEVnODn9ayWAw81QGI +id77DlK5BwGcJI79icF1dbfODRpAzuW/lmnNFQ9f8lQ +-> ssh-ed25519 5/zT0w +HimE5fwHyw00Mrl+A0OXw3unuqrRL7xBXsn3kLRFy8 +PoVKwvqyWGlPBaQ1ZTGd9gK1kc5w18z7hPJp1GAbZ1E +-> ssh-ed25519 TCgorQ oFe93M5WY5ovlHaNLBwA8LMRcydHtIt66IRzCmnxyQ4 +ni+pTTEFop45tAEBUz6zne9Xgi42+gMTdoVnAQoZUls +-> ssh-ed25519 d3WGuA g3Ku++27IcH9g3xa5NqXz1itzMkIg+qoVo+yX25K1Wk +Li5Ni2LbyFL8Bv/yCY5pwvK3/y5bS/quMvxnlwu2/7g +-> ssh-ed25519 YIaSKQ 2SMv68txiKxrx+fs2+zMMCEa2SP4appPHlBZgPRRDH8 +xUp17uDntP1VUqrKMh0Hj73TcJh2o2LT7jaR4q7PVjY +-> e['-grease +ZqwKkCj0096w+J1bHZS2kubJ3egBdMrxFVE12g7AMsKSmq6bC1HyWsFJ2ZMNNVX3 +L04 +--- C5zj/EiQIzWfVY4XJp0eNPTfLFXud+cMOBR9/43e1jA +G|HhGrVɲX9UՁ +z"Z-{KzލMj%*h^VEZ\V>uY֣&DB) \ No newline at end of file diff --git a/modules/ascensions/default.nix b/modules/ascensions/default.nix index 4503512..ea5bebc 100644 --- a/modules/ascensions/default.nix +++ b/modules/ascensions/default.nix @@ -85,6 +85,7 @@ in targetLevel = toString (length incantations); in { description = "Ascension for ${name}"; + wantedBy = [ "multi-user.target" ]; inherit (asc) requiredBy before; after = asc.after ++ (lib.optional asc.distributed "consul.service"); serviceConfig.Type = "oneshot"; diff --git a/modules/consul-distributed-services/default.nix b/modules/consul-distributed-services/default.nix index 1297a84..61f73e1 100644 --- a/modules/consul-distributed-services/default.nix +++ b/modules/consul-distributed-services/default.nix @@ -4,6 +4,9 @@ with lib; let consul = config.services.consul.package; + + consulCfg = config.services.consul.extraConfig; + consulHttpAddr = "${consulCfg.addresses.http or "127.0.0.1"}:${toString (consulCfg.ports.http or 8500)}"; in { options.systemd.services = mkOption { @@ -31,11 +34,19 @@ in ''${@} ''; + waitForConsul = pkgs.writeShellScript "wait-for-consul" '' + while ! ${consul}/bin/consul lock --name="pre-flight-check" --n=${toString cfg.replicas} --shell=false "$1" ${pkgs.coreutils}/bin/true; do + sleep 1 + done + ''; + hasSpecialPrefix = elem (substring 0 1 ExecStart) [ "@" "-" ":" "+" "!" ]; in assert !hasSpecialPrefix; pkgs.writeTextDir "etc/systemd/system/${n}.service.d/distributed.conf" '' [Service] + ExecStartPre=${waitForConsul} 'services/${n}%i' ExecStart= ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart} + Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}" ${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"} ${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"} '')) diff --git a/modules/consul-service-registry/default.nix b/modules/consul-service-registry/default.nix index 317066f..4257ca2 100644 --- a/modules/consul-service-registry/default.nix +++ b/modules/consul-service-registry/default.nix @@ -7,13 +7,18 @@ let consul = "${config.services.consul.package}/bin/consul"; + consulCfg = config.services.consul.extraConfig; + consulHttpAddr = "${consulCfg.addresses.http or "127.0.0.1"}:${toString (consulCfg.ports.http or 8500)}"; + consulRegisterScript = pkgs.writeShellScript "consul-register" '' + export CONSUL_HTTP_ADDR='${consulHttpAddr}' while ! ${consul} services register "$1"; do sleep 1 done ''; consulDeregisterScript = pkgs.writeShellScript "consul-deregister" '' + export CONSUL_HTTP_ADDR='${consulHttpAddr}' for i in {1..5}; do if ${consul} services deregister "$1"; then break diff --git a/modules/external-storage/default.nix b/modules/external-storage/default.nix index 42789fd..dd68a0a 100644 --- a/modules/external-storage/default.nix +++ b/modules/external-storage/default.nix @@ -49,6 +49,10 @@ in "credentials=${config.age.secrets."cifsCredentials-${name}".path}" "dir_mode=0700" "file_mode=0600" + "uid=${toString ul.uid}" + "gid=${toString ul.gid}" + "forceuid" + "forcegid" "seal" "hard" "resilienthandles" diff --git a/modules/external-storage/underlay-type.nix b/modules/external-storage/underlay-type.nix index d29ea57..504dc21 100644 --- a/modules/external-storage/underlay-type.nix +++ b/modules/external-storage/underlay-type.nix @@ -28,5 +28,13 @@ with lib; type = types.path; default = "/"; }; + uid = mkOption { + type = types.int; + default = 0; + }; + gid = mkOption { + type = types.int; + default = 0; + }; }; } diff --git a/packages/checks/default.nix b/packages/checks/default.nix index 8b84917..744d645 100644 --- a/packages/checks/default.nix +++ b/packages/checks/default.nix @@ -7,6 +7,12 @@ inherit (self) nixosModules; }; + garage = pkgs.callPackage ./garage.nix { + inherit (self'.packages) garage; + inherit (self) nixosModules; + inherit (config) cluster; + }; + jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix { inherit (self'.packages) jellyfin; inherit (config) cluster; diff --git a/packages/checks/garage.nix b/packages/checks/garage.nix new file mode 100644 index 0000000..6b9e34c --- /dev/null +++ b/packages/checks/garage.nix @@ -0,0 +1,141 @@ +{ testers, nixosModules, cluster, garage }: + +testers.runNixOSTest { + name = "garage"; + + imports = [ + ./modules/consul.nix + ]; + + nodes = let + common = { config, lib, ... }: let + inherit (config.networking) hostName primaryIPAddress; + in { + imports = lib.flatten [ + ./modules/nixos/age-dummy-secrets.nix + nixosModules.ascensions + nixosModules.systemd-extras + nixosModules.consul-distributed-services + cluster.config.services.storage.nixos.garage + cluster.config.services.storage.nixos.garageInternal + ]; + config = { + _module.args = { + depot.packages = { inherit garage; }; + cluster.config = { + hostLinks.${hostName} = { + garageRpc.tuple = "${primaryIPAddress}:3901"; + garageS3.tuple = "${primaryIPAddress}:8080"; + }; + vars.meshNet.cidr = "192.168.0.0/16"; + }; + }; + environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + networking.firewall.allowedTCPPorts = [ 3901 8080 ]; + services.garage = { + settings.consul_discovery.consul_http_addr = lib.mkForce "http://consul:8500"; + layout.initial = lib.mkOverride 51 { + garage1 = { zone = "dc1"; capacity = 1000; }; + garage2 = { zone = "dc1"; capacity = 1000; }; + garage3 = { zone = "dc1"; capacity = 1000; }; + }; + }; + system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]); + specialisation.modifiedLayout = { + inheritParentConfig = true; + configuration = { + services.garage = { + layout.initial = lib.mkForce { + garage1 = { zone = "dc1"; capacity = 2000; }; + garage2 = { zone = "dc1"; capacity = 1000; }; + garage3 = { zone = "dc1"; capacity = 1000; }; + }; + keys.testKey.allow.createBucket = true; + buckets = { + bucket1 = { + allow.testKey = [ "read" "write" ]; + quotas = { + maxObjects = 300; + maxSize = 400 * 1024 * 1024; + }; + }; + bucket2 = { + allow.testKey = [ "read" ]; + }; + }; + }; + system.ascensions.garage-layout.incantations = lib.mkForce (i: [ + (i.runGarage '' + garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)" + garage layout apply --version 2 + '') + ]); + }; + }; + }; + }; + in { + garage1.imports = [ common ]; + garage2.imports = [ common ]; + garage3.imports = [ common ]; + }; + + testScript = { nodes, ... }: /*python*/ '' + nodes = [garage1, garage2, garage3] + + start_all() + + with subtest("should bootstrap new cluster"): + for node in nodes: + node.wait_for_unit("garage.service") + + for node in nodes: + node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") + + with subtest("should apply new layout with ascension"): + for node in nodes: + node.wait_until_succeeds('test "$(systemctl is-active ascend-garage-layout)" != activating') + + for node in nodes: + node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test") + + for node in nodes: + node.wait_until_succeeds("garage layout show | grep -w 2000") + assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") + assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l") + + with subtest("should apply new layout from scratch"): + for node in nodes: + node.systemctl("stop garage.service") + node.succeed("rm -rf /var/lib/garage-metadata") + + for node in nodes: + node.systemctl("start garage.service") + + for node in nodes: + node.wait_for_unit("garage.service") + + for node in nodes: + node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'") + + for node in nodes: + node.wait_until_succeeds("garage layout show | grep -w 2000") + assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l") + assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l") + + with subtest("should create specified buckets and keys"): + for node in nodes: + node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating') + garage1.succeed("garage key list | grep testKey") + garage1.succeed("garage bucket list | grep bucket1") + garage1.succeed("garage bucket list | grep bucket2") + + with subtest("should delete unspecified buckets and keys"): + garage1.succeed("garage bucket create unwantedbucket") + garage1.succeed("garage key new --name unwantedkey") + garage1.succeed("systemctl restart garage-apply.service") + + garage1.fail("garage key list | grep unwantedkey") + garage1.fail("garage bucket list | grep unwantedbucket") + ''; +} diff --git a/packages/checks/modules/nixos/age-dummy-secrets.nix b/packages/checks/modules/nixos/age-dummy-secrets.nix new file mode 100644 index 0000000..05c120b --- /dev/null +++ b/packages/checks/modules/nixos/age-dummy-secrets.nix @@ -0,0 +1,33 @@ +{ config, lib, ... }: +with lib; + +let + t = { + string = default: mkOption { + type = types.str; + inherit default; + }; + }; +in + +{ + options.age.secrets = mkOption { + type = types.attrsOf (types.submodule ({ name, config, ... }: { + options = { + file = mkSinkUndeclaredOptions {}; + owner = t.string "root"; + group = t.string "root"; + mode = t.string "400"; + path = t.string "/etc/dummy-secrets/${name}"; + }; + })); + }; + config.environment.etc = mapAttrs' (name: secret: { + name = removePrefix "/etc/" secret.path; + value = mapAttrs (const mkDefault) { + user = secret.owner; + inherit (secret) mode group; + text = builtins.hashString "md5" name; + }; + }) config.age.secrets; +} diff --git a/packages/patched-derivations.nix b/packages/patched-derivations.nix index db9dcfb..3869da4 100644 --- a/packages/patched-derivations.nix +++ b/packages/patched-derivations.nix @@ -53,6 +53,8 @@ super: rec { forgejo = patch super.forgejo "patches/base/forgejo"; + garage = patch super.garage_0_8 "patches/base/garage"; + jellyfin = patch (super.jellyfin.override { ffmpeg = super.ffmpeg.override { withMfx = true; diff --git a/patches/base/garage/print-chill-pills.patch b/patches/base/garage/print-chill-pills.patch new file mode 100644 index 0000000..a0b4dd9 --- /dev/null +++ b/patches/base/garage/print-chill-pills.patch @@ -0,0 +1,264 @@ +diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs +index bbde204..7bed4b0 100644 +--- a/src/db/bin/convert.rs ++++ b/src/db/bin/convert.rs +@@ -4,6 +4,28 @@ use garage_db::*; + + use clap::Parser; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + /// K2V command line interface + #[derive(Parser, Debug)] + #[clap(author, version, about, long_about = None)] +diff --git a/src/db/lib.rs b/src/db/lib.rs +index 11cae4e..ffef3fa 100644 +--- a/src/db/lib.rs ++++ b/src/db/lib.rs +@@ -25,6 +25,18 @@ use std::sync::Arc; + + use err_derive::Error; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + #[derive(Clone)] + pub struct Db(pub(crate) Arc); + +diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs +index 0d73588..6bf4ecc 100644 +--- a/src/garage/cli/cmd.rs ++++ b/src/garage/cli/cmd.rs +@@ -13,6 +13,28 @@ use garage_model::helper::error::Error as HelperError; + use crate::admin::*; + use crate::cli::*; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + pub async fn cli_command_dispatch( + cmd: Command, + system_rpc_endpoint: &Endpoint, +diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs +index 20813f1..f4baea2 100644 +--- a/src/garage/cli/init.rs ++++ b/src/garage/cli/init.rs +@@ -2,6 +2,18 @@ use std::path::PathBuf; + + use garage_util::error::*; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + pub const READ_KEY_ERROR: &str = "Unable to read node key. It will be generated by your garage node the first time is it launched. Ensure that your garage node is currently running. (The node key is supposed to be stored in your metadata directory.)"; + + pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> { +diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs +index 3884bb9..ef55a66 100644 +--- a/src/garage/cli/layout.rs ++++ b/src/garage/cli/layout.rs +@@ -8,6 +8,28 @@ use garage_rpc::*; + + use crate::cli::*; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + pub async fn cli_layout_command_dispatch( + cmd: LayoutOperation, + system_rpc_endpoint: &Endpoint, +diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs +index 2c6be2f..db6f25d 100644 +--- a/src/garage/cli/util.rs ++++ b/src/garage/cli/util.rs +@@ -17,6 +17,28 @@ use garage_model::s3::version_table::Version; + + use crate::cli::structs::WorkerListOpt; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + pub fn print_bucket_list(bl: Vec) { + println!("List of buckets:"); + +diff --git a/src/k2v-client/bin/k2v-cli.rs b/src/k2v-client/bin/k2v-cli.rs +index cdd63cc..dfa4df4 100644 +--- a/src/k2v-client/bin/k2v-cli.rs ++++ b/src/k2v-client/bin/k2v-cli.rs +@@ -11,6 +11,28 @@ use rusoto_core::Region; + + use clap::{Parser, Subcommand}; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + /// K2V command line interface + #[derive(Parser, Debug)] + #[clap(author, version, about, long_about = None)] +diff --git a/src/rpc/layout.rs b/src/rpc/layout.rs +index 1030e3a..47eca49 100644 +--- a/src/rpc/layout.rs ++++ b/src/rpc/layout.rs +@@ -10,6 +10,28 @@ use garage_util::error::*; + + use crate::ring::*; + ++use std::io::Write; ++ ++macro_rules! println { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ writeln!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + /// The layout of the cluster, i.e. the list of roles + /// which are assigned to each cluster node + #[derive(Clone, Debug, Serialize, Deserialize)] +diff --git a/src/util/formater.rs b/src/util/formater.rs +index 2ea53eb..cc7d8a4 100644 +--- a/src/util/formater.rs ++++ b/src/util/formater.rs +@@ -1,3 +1,15 @@ ++use std::io::Write; ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + pub fn format_table_to_string(data: Vec) -> String { + let data = data + .iter() diff --git a/secrets.nix b/secrets.nix index 5b0b418..c3da200 100644 --- a/secrets.nix +++ b/secrets.nix @@ -44,7 +44,8 @@ in with hosts; "cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ]; "cluster/services/storage/secrets/heresy-encryption-key.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "cluster/services/storage/secrets/external-storage-encryption-key-prophet.age".publicKeys = max ++ map systemKeys [ prophet ]; - "cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ]; + "cluster/services/storage/secrets/garage-rpc-secret.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ]; + "cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ]; "cluster/services/wireguard/mesh-keys/checkmate.age".publicKeys = max ++ map systemKeys [ checkmate ]; "cluster/services/wireguard/mesh-keys/thunderskin.age".publicKeys = max ++ map systemKeys [ thunderskin ]; "cluster/services/wireguard/mesh-keys/VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ];