From f4779a851241871f14ed14ec477bcdd4b5ddcaea Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Sep 2023 02:50:09 +0200 Subject: [PATCH] cluster/services/storage: declarative garage keys and buckets --- cluster/services/storage/garage-options.nix | 211 +++++++++++++++++--- 1 file changed, 183 insertions(+), 28 deletions(-) diff --git a/cluster/services/storage/garage-options.nix b/cluster/services/storage/garage-options.nix index 2c1bbc1..c5e6509 100644 --- a/cluster/services/storage/garage-options.nix +++ b/cluster/services/storage/garage-options.nix @@ -1,9 +1,9 @@ -{ config, lib, ... }: +{ config, lib, pkgs, ... }: let cfg = config.services.garage; - garageShellLibrary = /*bash*/ '' + garageShellLibrary = pkgs.writeText "garage-shell-library.sh" '' getNodeId() { nodeId="" while [[ -z "$nodeId" ]]; do @@ -20,58 +20,213 @@ let sleep 1 done } + waitForGarageOperational() { + waitForGarage + while garage layout show | grep -qwm1 '^Current cluster layout version: 0'; do + sleep 1 + done + } + # FIXME: returns bogus empty string when one of the lists is empty + diffAdded() { + comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort) + } + diffRemoved() { + comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort) + } + # FIXME: this does not handle list items with spaces + listKeys() { + garage key list | tail -n +2 | grep -ow '[^ ]*$' || true + } + ensureKeys() { + old="$(listKeys)" + if [[ -z "$1" ]]; then + for key in $old; do + garage key delete --yes "$key" + done + elif [[ -z "$old" ]]; then + for key in $1; do + # don't print secret key + garage key new --name "$key" >/dev/null + echo Key "$key" was created. + done + else + diffAdded "$old" "$1" | while read key; do + # don't print secret key + garage key new --name "$key" >/dev/null + echo Key "$key" was created. + done + diffRemoved "$old" "$1" | while read key; do + garage key delete --yes "$key" + done + fi + } + listBuckets() { + garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true + } + ensureBuckets() { + old="$(listBuckets)" + if [[ -z "$1" ]]; then + for bucket in $old; do + garage bucket delete --yes "$bucket" + done + elif [[ -z "$old" ]]; then + for bucket in $1; do + garage bucket create "$bucket" + done + else + diffAdded "$old" "$1" | while read bucket; do + garage bucket create "$bucket" + done + diffRemoved "$old" "$1" | while read bucket; do + garage bucket delete --yes "$bucket" + done + fi + } ''; in { - options.services.garage.layout = { - initial = lib.mkOption { + options.services.garage = with lib; { + layout.initial = mkOption { default = {}; - type = with lib.types; attrsOf (submodule { + type = with types; attrsOf (submodule { options = { - zone = lib.mkOption { - type = lib.types.str; + zone = mkOption { + type = types.str; }; - capacity = lib.mkOption { - type = lib.types.ints.positive; + capacity = mkOption { + type = types.ints.positive; }; }; }); }; + keys = mkOption { + type = with types; attrsOf (submodule { + options.allow = { + createBucket = mkOption { + description = "Allow the key to create new buckets."; + type = bool; + default = false; + }; + }; + }); + default = {}; + }; + buckets = mkOption { + type = with types; attrsOf (submodule { + options = { + allow = mkOption { + description = "List of permissions to grant on this bucket, grouped by key name."; + type = attrsOf (listOf (enum [ "read" "write" "owner" ])); + default = {}; + }; + quotas = { + maxObjects = mkOption { + description = "Maximum number of objects in this bucket. Null for unlimited."; + type = nullOr ints.positive; + default = null; + }; + maxSize = mkOption { + description = "Maximum size of this bucket in bytes. Null for unlimited."; + type = nullOr ints.positive; + default = null; + }; + }; + }; + }); + default = {}; + }; }; config = { system.extraIncantations = { runGarage = i: script: i.execShellWith [ config.services.garage.package ] '' - ${garageShellLibrary} + source ${garageShellLibrary} waitForGarage ${script} ''; }; - systemd.services.garage-layout-init = { - distributed.enable = true; - wantedBy = [ "garage.service" ]; - after = [ "garage.service" ]; - path = [ config.services.garage.package ]; + systemd.services = { + garage-layout-init = { + distributed.enable = true; + wantedBy = [ "garage.service" "multi-user.target" ]; + wants = [ "garage.service" ]; + after = [ "garage.service" ]; + path = [ config.services.garage.package ]; - serviceConfig = { - TimeoutStartSec = "1800s"; + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = "1800s"; + Restart = "on-failure"; + RestartSec = "10s"; + }; + script = '' + source ${garageShellLibrary} + waitForGarage + + if [[ "$(garage layout show | grep -m1 '^Current cluster layout version:' | cut -d: -f2 | tr -d ' ')" != "0" ]]; then + exit 0 + fi + + ${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: layout: '' + garage layout assign -z '${layout.zone}' -c '${toString layout.capacity}' "$(getNodeId '${name}')" + '') cfg.layout.initial)} + + garage layout apply --version 1 + ''; }; - script = '' - ${garageShellLibrary} - waitForGarage + garage-apply = { + distributed.enable = true; + wantedBy = [ "garage.service" "multi-user.target" ]; + wants = [ "garage.service" ]; + after = [ "garage.service" "garage-layout-init.service" ]; + path = [ config.services.garage.package ]; - if [[ "$(garage layout show | grep -m1 '^Current cluster layout version:' | cut -d: -f2 | tr -d ' ')" != "0" ]]; then - exit 0 - fi + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = "1800s"; + Restart = "on-failure"; + RestartSec = "10s"; + }; + script = '' + source ${garageShellLibrary} + waitForGarageOperational - ${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: layout: '' - garage layout assign -z '${layout.zone}' -c '${toString layout.capacity}' "$(getNodeId '${name}')" - '') cfg.layout.initial)} + ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}' + ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}' - garage layout apply --version 1 - ''; + # key permissions + ${lib.pipe cfg.keys [ + (lib.mapAttrsToList (key: kCfg: '' + garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null + '')) + (lib.concatStringsSep "\n") + ]} + + # bucket permissions + ${lib.pipe cfg.buckets [ + (lib.mapAttrsToList (bucket: bCfg: + lib.mapAttrsToList (key: perms: '' + garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)} + garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))} + '') bCfg.allow + )) + lib.flatten + (lib.concatStringsSep "\n") + ]} + + # bucket quotas + ${lib.pipe cfg.buckets [ + (lib.mapAttrsToList (bucket: bCfg: '' + garage bucket set-quotas '${bucket}' \ + --max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \ + --max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}' + '')) + (lib.concatStringsSep "\n") + ]} + ''; + }; }; }; }