GitHub
commit
2ca2094d3a
19 changed files with 879 additions and 13 deletions
|
|
@ -1,12 +1,38 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
let
|
||||
meshIpForNode = name: config.vars.mesh.${name}.meshIp;
|
||||
in
|
||||
|
||||
{
|
||||
services.storage = {
|
||||
nodes = {
|
||||
external = [ "prophet" ];
|
||||
heresy = [ "VEGAS" ];
|
||||
garage = [ "checkmate" "prophet" "VEGAS" ];
|
||||
garageInternal = [ "VEGAS" ];
|
||||
garageExternal = [ "checkmate" "prophet" ];
|
||||
};
|
||||
nixos = {
|
||||
external = [ ./external.nix ];
|
||||
heresy = [ ./heresy.nix ];
|
||||
garage = [
|
||||
./garage.nix
|
||||
./garage-options.nix
|
||||
./garage-layout.nix
|
||||
];
|
||||
garageInternal = [ ./garage-internal.nix ];
|
||||
garageExternal = [ ./garage-external.nix ];
|
||||
};
|
||||
};
|
||||
|
||||
hostLinks = lib.genAttrs config.services.storage.nodes.garage (name: {
|
||||
garageRpc = {
|
||||
ipv4 = meshIpForNode name;
|
||||
};
|
||||
garageS3 = {
|
||||
protocol = "http";
|
||||
ipv4 = meshIpForNode name;
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
|
|||
15
cluster/services/storage/garage-external.nix
Normal file
15
cluster/services/storage/garage-external.nix
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{ config, ... }:
|
||||
|
||||
{
|
||||
services.external-storage = {
|
||||
underlays.garage = {
|
||||
subUser = "sub1";
|
||||
credentialsFile = ./secrets/storage-box-credentials.age;
|
||||
path = "/garage/${config.networking.hostName}";
|
||||
inherit (config.users.users.garage) uid;
|
||||
inherit (config.users.groups.garage) gid;
|
||||
};
|
||||
};
|
||||
|
||||
services.garage.settings.data_dir = config.services.external-storage.underlays.garage.mountpoint;
|
||||
}
|
||||
11
cluster/services/storage/garage-internal.nix
Normal file
11
cluster/services/storage/garage-internal.nix
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
let
|
||||
dataDir = "/srv/storage/private/garage";
|
||||
in
|
||||
|
||||
{
|
||||
systemd.tmpfiles.rules = [
|
||||
"d '${dataDir}' 0700 garage garage -"
|
||||
];
|
||||
|
||||
services.garage.settings.data_dir = dataDir;
|
||||
}
|
||||
14
cluster/services/storage/garage-layout.nix
Normal file
14
cluster/services/storage/garage-layout.nix
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
system.ascensions.garage-layout = {
|
||||
distributed = true;
|
||||
requiredBy = [ "garage.service" ];
|
||||
after = [ "garage.service" "garage-layout-init.service" ];
|
||||
incantations = i: [ ];
|
||||
};
|
||||
|
||||
services.garage.layout.initial = {
|
||||
checkmate = { zone = "eu-central"; capacity = 1000; };
|
||||
prophet = { zone = "eu-central"; capacity = 1000; };
|
||||
VEGAS = { zone = "eu-central"; capacity = 1000; };
|
||||
};
|
||||
}
|
||||
232
cluster/services/storage/garage-options.nix
Normal file
232
cluster/services/storage/garage-options.nix
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.garage;
|
||||
|
||||
garageShellLibrary = pkgs.writeText "garage-shell-library.sh" ''
|
||||
getNodeId() {
|
||||
nodeId=""
|
||||
while [[ -z "$nodeId" ]]; do
|
||||
nodeId="$(garage status | grep 'NO ROLE ASSIGNED' | grep -wm1 "$1" | cut -d' ' -f1)"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Waiting for node $1 to appear..." 2>/dev/null
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
echo "$nodeId"
|
||||
}
|
||||
waitForGarage() {
|
||||
while ! garage status >/dev/null 2>/dev/null; do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
waitForGarageOperational() {
|
||||
waitForGarage
|
||||
while garage layout show | grep -qwm1 '^Current cluster layout version: 0'; do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
# FIXME: returns bogus empty string when one of the lists is empty
|
||||
diffAdded() {
|
||||
comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
|
||||
}
|
||||
diffRemoved() {
|
||||
comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
|
||||
}
|
||||
# FIXME: this does not handle list items with spaces
|
||||
listKeys() {
|
||||
garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
|
||||
}
|
||||
ensureKeys() {
|
||||
old="$(listKeys)"
|
||||
if [[ -z "$1" ]]; then
|
||||
for key in $old; do
|
||||
garage key delete --yes "$key"
|
||||
done
|
||||
elif [[ -z "$old" ]]; then
|
||||
for key in $1; do
|
||||
# don't print secret key
|
||||
garage key new --name "$key" >/dev/null
|
||||
echo Key "$key" was created.
|
||||
done
|
||||
else
|
||||
diffAdded "$old" "$1" | while read key; do
|
||||
# don't print secret key
|
||||
garage key new --name "$key" >/dev/null
|
||||
echo Key "$key" was created.
|
||||
done
|
||||
diffRemoved "$old" "$1" | while read key; do
|
||||
garage key delete --yes "$key"
|
||||
done
|
||||
fi
|
||||
}
|
||||
listBuckets() {
|
||||
garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
|
||||
}
|
||||
ensureBuckets() {
|
||||
old="$(listBuckets)"
|
||||
if [[ -z "$1" ]]; then
|
||||
for bucket in $old; do
|
||||
garage bucket delete --yes "$bucket"
|
||||
done
|
||||
elif [[ -z "$old" ]]; then
|
||||
for bucket in $1; do
|
||||
garage bucket create "$bucket"
|
||||
done
|
||||
else
|
||||
diffAdded "$old" "$1" | while read bucket; do
|
||||
garage bucket create "$bucket"
|
||||
done
|
||||
diffRemoved "$old" "$1" | while read bucket; do
|
||||
garage bucket delete --yes "$bucket"
|
||||
done
|
||||
fi
|
||||
}
|
||||
'';
|
||||
in
|
||||
|
||||
{
|
||||
options.services.garage = with lib; {
|
||||
layout.initial = mkOption {
|
||||
default = {};
|
||||
type = with types; attrsOf (submodule {
|
||||
options = {
|
||||
zone = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
capacity = mkOption {
|
||||
type = types.ints.positive;
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
keys = mkOption {
|
||||
type = with types; attrsOf (submodule {
|
||||
options.allow = {
|
||||
createBucket = mkOption {
|
||||
description = "Allow the key to create new buckets.";
|
||||
type = bool;
|
||||
default = false;
|
||||
};
|
||||
};
|
||||
});
|
||||
default = {};
|
||||
};
|
||||
buckets = mkOption {
|
||||
type = with types; attrsOf (submodule {
|
||||
options = {
|
||||
allow = mkOption {
|
||||
description = "List of permissions to grant on this bucket, grouped by key name.";
|
||||
type = attrsOf (listOf (enum [ "read" "write" "owner" ]));
|
||||
default = {};
|
||||
};
|
||||
quotas = {
|
||||
maxObjects = mkOption {
|
||||
description = "Maximum number of objects in this bucket. Null for unlimited.";
|
||||
type = nullOr ints.positive;
|
||||
default = null;
|
||||
};
|
||||
maxSize = mkOption {
|
||||
description = "Maximum size of this bucket in bytes. Null for unlimited.";
|
||||
type = nullOr ints.positive;
|
||||
default = null;
|
||||
};
|
||||
};
|
||||
};
|
||||
});
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
system.extraIncantations = {
|
||||
runGarage = i: script: i.execShellWith [ config.services.garage.package ] ''
|
||||
source ${garageShellLibrary}
|
||||
waitForGarage
|
||||
${script}
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services = {
|
||||
garage-layout-init = {
|
||||
distributed.enable = true;
|
||||
wantedBy = [ "garage.service" "multi-user.target" ];
|
||||
wants = [ "garage.service" ];
|
||||
after = [ "garage.service" ];
|
||||
path = [ config.services.garage.package ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
TimeoutStartSec = "1800s";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "10s";
|
||||
};
|
||||
script = ''
|
||||
source ${garageShellLibrary}
|
||||
waitForGarage
|
||||
|
||||
if [[ "$(garage layout show | grep -m1 '^Current cluster layout version:' | cut -d: -f2 | tr -d ' ')" != "0" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: layout: ''
|
||||
garage layout assign -z '${layout.zone}' -c '${toString layout.capacity}' "$(getNodeId '${name}')"
|
||||
'') cfg.layout.initial)}
|
||||
|
||||
garage layout apply --version 1
|
||||
'';
|
||||
};
|
||||
garage-apply = {
|
||||
distributed.enable = true;
|
||||
wantedBy = [ "garage.service" "multi-user.target" ];
|
||||
wants = [ "garage.service" ];
|
||||
after = [ "garage.service" "garage-layout-init.service" ];
|
||||
path = [ config.services.garage.package ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
TimeoutStartSec = "1800s";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "10s";
|
||||
};
|
||||
script = ''
|
||||
source ${garageShellLibrary}
|
||||
waitForGarageOperational
|
||||
|
||||
ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
|
||||
ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
|
||||
|
||||
# key permissions
|
||||
${lib.pipe cfg.keys [
|
||||
(lib.mapAttrsToList (key: kCfg: ''
|
||||
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket permissions
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg:
|
||||
lib.mapAttrsToList (key: perms: ''
|
||||
garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
|
||||
garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
|
||||
'') bCfg.allow
|
||||
))
|
||||
lib.flatten
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket quotas
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg: ''
|
||||
garage bucket set-quotas '${bucket}' \
|
||||
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
|
||||
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
71
cluster/services/storage/garage.nix
Normal file
71
cluster/services/storage/garage.nix
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
{ cluster, config, depot, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster.config) vars;
|
||||
inherit (config.networking) hostName;
|
||||
|
||||
links = cluster.config.hostLinks.${hostName};
|
||||
|
||||
cfg = config.services.garage;
|
||||
|
||||
# fixed uid so we know how to mount the underlay
|
||||
uid = 777;
|
||||
gid = 777;
|
||||
in
|
||||
|
||||
{
|
||||
age.secrets.garageRpcSecret = {
|
||||
file = ./secrets/garage-rpc-secret.age;
|
||||
owner = "garage";
|
||||
group = "garage";
|
||||
};
|
||||
|
||||
services.garage = {
|
||||
enable = true;
|
||||
package = depot.packages.garage;
|
||||
settings = {
|
||||
replication_mode = 3;
|
||||
block_size = 16 * 1024 * 1024;
|
||||
db_engine = "lmdb";
|
||||
metadata_dir = "/var/lib/garage-metadata";
|
||||
rpc_bind_addr = links.garageRpc.tuple;
|
||||
rpc_public_addr = links.garageRpc.tuple;
|
||||
rpc_secret_file = config.age.secrets.garageRpcSecret.path;
|
||||
consul_discovery = {
|
||||
consul_http_addr = "http://127.0.0.1:8500";
|
||||
api = "agent";
|
||||
service_name = "garage-discovery";
|
||||
};
|
||||
s3_api = {
|
||||
api_bind_addr = links.garageS3.tuple;
|
||||
s3_region = "us-east-1";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
users = {
|
||||
users.garage = {
|
||||
inherit uid;
|
||||
group = "garage";
|
||||
};
|
||||
groups.garage = {
|
||||
inherit gid;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.garage = {
|
||||
unitConfig = {
|
||||
RequiresMountsFor = [ cfg.settings.data_dir ];
|
||||
};
|
||||
serviceConfig = {
|
||||
IPAddressDeny = [ "any" ];
|
||||
IPAddressAllow = [ "127.0.0.1/8" vars.meshNet.cidr ];
|
||||
DynamicUser = false;
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = true;
|
||||
User = "garage";
|
||||
Group = "garage";
|
||||
StateDirectory = lib.removePrefix "/var/lib/" cfg.settings.metadata_dir;
|
||||
};
|
||||
};
|
||||
}
|
||||
17
cluster/services/storage/secrets/garage-rpc-secret.age
Normal file
17
cluster/services/storage/secrets/garage-rpc-secret.age
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 NO562A WymFH+09McLeT2+o487RiKzExeG7QKTwwuERimfVygs
|
||||
dLTYSgo35f3cvhRWzhnutjSSRChBhhdX+87Yd6H6A5w
|
||||
-> ssh-ed25519 5/zT0w izNXf7mqdiDyc6AN6kbQfMHEtGxiI6+DnyiVxaSqFDo
|
||||
mXu7h0oY4UgiMojcNKedWXwvnQJmq4wyFkE1ZVTlXSk
|
||||
-> ssh-ed25519 TCgorQ 7ueU0wcT7A5bI6/JVKuIAj/jhh7TuVEfDCullS/Ws0c
|
||||
nri4g0jk8hdGMYgxjarklPBmzEBfXv2tgys8xPJ0UVI
|
||||
-> ssh-ed25519 d3WGuA 3LNotYdeRf4QP7bEdjx9NHInFVgRCFq91xjixu96mBI
|
||||
Gj0LYo6dTVIcSOXTtqdug+zX0l6UIigBu5obEgIMK00
|
||||
-> ssh-ed25519 YIaSKQ DkCXHGHVP92yEBKlELG/mWPkbq4lXoAmFVXmemC6Sxw
|
||||
YE4kSaNFVcS3txw94o5WTOKXe6xYbQALyzxBWDlZsCA
|
||||
-> ;",X-grease &l-# ,k
|
||||
FUMCq19iC2wN9U5NBYrHmCdUKB+p5AOnYGiA3LeWTJK3f899Bht0ZbSl7kL1l4w8
|
||||
4+n1ZbUITvPNOg
|
||||
--- Fri66T6QtwEkou+ThWbOy0m651htF4yedOH6tL0VX0E
|
||||
…Ѹ·$ýª€ð\÷»»½Ï%¶eè¸].IÍí »+G<>Ó˜ É
|
||||
A–/kÀÃ8¿¯ÇiT“¿#•ê{é4µŽò@¿èh8 ¾I \¹Mgµì·Kö¾ÕGx×è°aÿð§qO
|
||||
|
|
@ -1,13 +1,17 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 NO562A zJvkOwBBlON92XKax+VTvXyqGFajgwzt091jNrRzjjI
|
||||
1a8/gXx4DQu03Rewebxdz5BpEi3zI7DXTOcY2OLIUck
|
||||
-> ssh-ed25519 5/zT0w cR4KCvOxvEV5KhEL+4HEcwKv5dKC4dT+yVok8P6Y/1c
|
||||
UCPh+WJ9opCFkDZgGBzyep8xZ58P6jLP1thMOpsx5rQ
|
||||
-> ssh-ed25519 d3WGuA nwgv+GeEw1Yc6gYCN39zfDyFg/NGxazU9tMkykmwhBc
|
||||
b+1EyRRSCvUWdMKcsEZA5BAnqQHO703Bwp5xtLPYok4
|
||||
-> ssh-ed25519 YIaSKQ FtSyO68mVkMEbMZV9w/H4tj7ms7gKOT+jB9n5MG5NAE
|
||||
B8Id7KRU9a19oWfmDcCN4zUb04AMZ4Z+AETyCvxeC7c
|
||||
-> zAL;6ee-grease O]^kl 0`0G tf+sVH= `+K,h=HL
|
||||
ZO4H8I1upQszxI6GBalw1hUhSAhRRhhePbE
|
||||
--- lfbOCcNCO7rc1os2qEc9Bj0s6+bxJujVzmdL3lWdm3g
|
||||
‡ÎeýÒm1ùuUVÈÏò#ÍjTóq–‘e=2Ìo¢ò9y2«3 õ• á?ù©jT1©‹¤œüGBAôïÇ_ü¹À3øÂg1-rL)Ë–gR <09>
|
||||
-> ssh-ed25519 NO562A wZnh+3ROEclIIvMM4EhbUqytTwEVnODn9ayWAw81QGI
|
||||
id77DlK5BwGcJI79icF1dbfODRpAzuW/lmnNFQ9f8lQ
|
||||
-> ssh-ed25519 5/zT0w +HimE5fwHyw00Mrl+A0OXw3unuqrRL7xBXsn3kLRFy8
|
||||
PoVKwvqyWGlPBaQ1ZTGd9gK1kc5w18z7hPJp1GAbZ1E
|
||||
-> ssh-ed25519 TCgorQ oFe93M5WY5ovlHaNLBwA8LMRcydHtIt66IRzCmnxyQ4
|
||||
ni+pTTEFop45tAEBUz6zne9Xgi42+gMTdoVnAQoZUls
|
||||
-> ssh-ed25519 d3WGuA g3Ku++27IcH9g3xa5NqXz1itzMkIg+qoVo+yX25K1Wk
|
||||
Li5Ni2LbyFL8Bv/yCY5pwvK3/y5bS/quMvxnlwu2/7g
|
||||
-> ssh-ed25519 YIaSKQ 2SMv68txiKxrx+fs2+zMMCEa2SP4appPHlBZgPRRDH8
|
||||
xUp17uDntP1VUqrKMh0Hj73TcJh2o2LT7jaR4q7PVjY
|
||||
-> e['-grease
|
||||
ZqwKkCj0096w+J1bHZS2kubJ3egBdMrxFVE12g7AMsKSmq6bC1HyWsFJ2ZMNNVX3
|
||||
L04
|
||||
--- C5zj/EiQIzWfVY4XJp0eNPTfLFXud+cMOBR9/43e1jA
|
||||
³õG|HhG…ˆ“r–˜VËɲX9÷UÕ<55>
|
||||
Ýz˜"¥»Zƒ-“{KzÞ<>µM‡j%*h^V¶EŽÌÒZ¶\¡V>óuYÖ£ &ËýD„BŠ)
|
||||
|
|
@ -85,6 +85,7 @@ in
|
|||
targetLevel = toString (length incantations);
|
||||
in {
|
||||
description = "Ascension for ${name}";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
inherit (asc) requiredBy before;
|
||||
after = asc.after ++ (lib.optional asc.distributed "consul.service");
|
||||
serviceConfig.Type = "oneshot";
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ with lib;
|
|||
|
||||
let
|
||||
consul = config.services.consul.package;
|
||||
|
||||
consulCfg = config.services.consul.extraConfig;
|
||||
consulHttpAddr = "${consulCfg.addresses.http or "127.0.0.1"}:${toString (consulCfg.ports.http or 8500)}";
|
||||
in
|
||||
{
|
||||
options.systemd.services = mkOption {
|
||||
|
|
@ -31,11 +34,19 @@ in
|
|||
''${@}
|
||||
'';
|
||||
|
||||
waitForConsul = pkgs.writeShellScript "wait-for-consul" ''
|
||||
while ! ${consul}/bin/consul lock --name="pre-flight-check" --n=${toString cfg.replicas} --shell=false "$1" ${pkgs.coreutils}/bin/true; do
|
||||
sleep 1
|
||||
done
|
||||
'';
|
||||
|
||||
hasSpecialPrefix = elem (substring 0 1 ExecStart) [ "@" "-" ":" "+" "!" ];
|
||||
in assert !hasSpecialPrefix; pkgs.writeTextDir "etc/systemd/system/${n}.service.d/distributed.conf" ''
|
||||
[Service]
|
||||
ExecStartPre=${waitForConsul} 'services/${n}%i'
|
||||
ExecStart=
|
||||
ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
|
||||
Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
|
||||
${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
|
||||
${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
|
||||
''))
|
||||
|
|
|
|||
|
|
@ -7,13 +7,18 @@ let
|
|||
|
||||
consul = "${config.services.consul.package}/bin/consul";
|
||||
|
||||
consulCfg = config.services.consul.extraConfig;
|
||||
consulHttpAddr = "${consulCfg.addresses.http or "127.0.0.1"}:${toString (consulCfg.ports.http or 8500)}";
|
||||
|
||||
consulRegisterScript = pkgs.writeShellScript "consul-register" ''
|
||||
export CONSUL_HTTP_ADDR='${consulHttpAddr}'
|
||||
while ! ${consul} services register "$1"; do
|
||||
sleep 1
|
||||
done
|
||||
'';
|
||||
|
||||
consulDeregisterScript = pkgs.writeShellScript "consul-deregister" ''
|
||||
export CONSUL_HTTP_ADDR='${consulHttpAddr}'
|
||||
for i in {1..5}; do
|
||||
if ${consul} services deregister "$1"; then
|
||||
break
|
||||
|
|
|
|||
|
|
@ -49,6 +49,10 @@ in
|
|||
"credentials=${config.age.secrets."cifsCredentials-${name}".path}"
|
||||
"dir_mode=0700"
|
||||
"file_mode=0600"
|
||||
"uid=${toString ul.uid}"
|
||||
"gid=${toString ul.gid}"
|
||||
"forceuid"
|
||||
"forcegid"
|
||||
"seal"
|
||||
"hard"
|
||||
"resilienthandles"
|
||||
|
|
|
|||
|
|
@ -28,5 +28,13 @@ with lib;
|
|||
type = types.path;
|
||||
default = "/";
|
||||
};
|
||||
uid = mkOption {
|
||||
type = types.int;
|
||||
default = 0;
|
||||
};
|
||||
gid = mkOption {
|
||||
type = types.int;
|
||||
default = 0;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,12 @@
|
|||
inherit (self) nixosModules;
|
||||
};
|
||||
|
||||
garage = pkgs.callPackage ./garage.nix {
|
||||
inherit (self'.packages) garage;
|
||||
inherit (self) nixosModules;
|
||||
inherit (config) cluster;
|
||||
};
|
||||
|
||||
jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix {
|
||||
inherit (self'.packages) jellyfin;
|
||||
inherit (config) cluster;
|
||||
|
|
|
|||
141
packages/checks/garage.nix
Normal file
141
packages/checks/garage.nix
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
{ testers, nixosModules, cluster, garage }:
|
||||
|
||||
testers.runNixOSTest {
|
||||
name = "garage";
|
||||
|
||||
imports = [
|
||||
./modules/consul.nix
|
||||
];
|
||||
|
||||
nodes = let
|
||||
common = { config, lib, ... }: let
|
||||
inherit (config.networking) hostName primaryIPAddress;
|
||||
in {
|
||||
imports = lib.flatten [
|
||||
./modules/nixos/age-dummy-secrets.nix
|
||||
nixosModules.ascensions
|
||||
nixosModules.systemd-extras
|
||||
nixosModules.consul-distributed-services
|
||||
cluster.config.services.storage.nixos.garage
|
||||
cluster.config.services.storage.nixos.garageInternal
|
||||
];
|
||||
config = {
|
||||
_module.args = {
|
||||
depot.packages = { inherit garage; };
|
||||
cluster.config = {
|
||||
hostLinks.${hostName} = {
|
||||
garageRpc.tuple = "${primaryIPAddress}:3901";
|
||||
garageS3.tuple = "${primaryIPAddress}:8080";
|
||||
};
|
||||
vars.meshNet.cidr = "192.168.0.0/16";
|
||||
};
|
||||
};
|
||||
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
networking.firewall.allowedTCPPorts = [ 3901 8080 ];
|
||||
services.garage = {
|
||||
settings.consul_discovery.consul_http_addr = lib.mkForce "http://consul:8500";
|
||||
layout.initial = lib.mkOverride 51 {
|
||||
garage1 = { zone = "dc1"; capacity = 1000; };
|
||||
garage2 = { zone = "dc1"; capacity = 1000; };
|
||||
garage3 = { zone = "dc1"; capacity = 1000; };
|
||||
};
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
|
||||
specialisation.modifiedLayout = {
|
||||
inheritParentConfig = true;
|
||||
configuration = {
|
||||
services.garage = {
|
||||
layout.initial = lib.mkForce {
|
||||
garage1 = { zone = "dc1"; capacity = 2000; };
|
||||
garage2 = { zone = "dc1"; capacity = 1000; };
|
||||
garage3 = { zone = "dc1"; capacity = 1000; };
|
||||
};
|
||||
keys.testKey.allow.createBucket = true;
|
||||
buckets = {
|
||||
bucket1 = {
|
||||
allow.testKey = [ "read" "write" ];
|
||||
quotas = {
|
||||
maxObjects = 300;
|
||||
maxSize = 400 * 1024 * 1024;
|
||||
};
|
||||
};
|
||||
bucket2 = {
|
||||
allow.testKey = [ "read" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
|
||||
(i.runGarage ''
|
||||
garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
|
||||
garage layout apply --version 2
|
||||
'')
|
||||
]);
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in {
|
||||
garage1.imports = [ common ];
|
||||
garage2.imports = [ common ];
|
||||
garage3.imports = [ common ];
|
||||
};
|
||||
|
||||
testScript = { nodes, ... }: /*python*/ ''
|
||||
nodes = [garage1, garage2, garage3]
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap new cluster"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
with subtest("should apply new layout with ascension"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl is-active ascend-garage-layout)" != activating')
|
||||
|
||||
for node in nodes:
|
||||
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should apply new layout from scratch"):
|
||||
for node in nodes:
|
||||
node.systemctl("stop garage.service")
|
||||
node.succeed("rm -rf /var/lib/garage-metadata")
|
||||
|
||||
for node in nodes:
|
||||
node.systemctl("start garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should create specified buckets and keys"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
|
||||
garage1.succeed("garage key list | grep testKey")
|
||||
garage1.succeed("garage bucket list | grep bucket1")
|
||||
garage1.succeed("garage bucket list | grep bucket2")
|
||||
|
||||
with subtest("should delete unspecified buckets and keys"):
|
||||
garage1.succeed("garage bucket create unwantedbucket")
|
||||
garage1.succeed("garage key new --name unwantedkey")
|
||||
garage1.succeed("systemctl restart garage-apply.service")
|
||||
|
||||
garage1.fail("garage key list | grep unwantedkey")
|
||||
garage1.fail("garage bucket list | grep unwantedbucket")
|
||||
'';
|
||||
}
|
||||
33
packages/checks/modules/nixos/age-dummy-secrets.nix
Normal file
33
packages/checks/modules/nixos/age-dummy-secrets.nix
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
{ config, lib, ... }:
|
||||
with lib;
|
||||
|
||||
let
|
||||
t = {
|
||||
string = default: mkOption {
|
||||
type = types.str;
|
||||
inherit default;
|
||||
};
|
||||
};
|
||||
in
|
||||
|
||||
{
|
||||
options.age.secrets = mkOption {
|
||||
type = types.attrsOf (types.submodule ({ name, config, ... }: {
|
||||
options = {
|
||||
file = mkSinkUndeclaredOptions {};
|
||||
owner = t.string "root";
|
||||
group = t.string "root";
|
||||
mode = t.string "400";
|
||||
path = t.string "/etc/dummy-secrets/${name}";
|
||||
};
|
||||
}));
|
||||
};
|
||||
config.environment.etc = mapAttrs' (name: secret: {
|
||||
name = removePrefix "/etc/" secret.path;
|
||||
value = mapAttrs (const mkDefault) {
|
||||
user = secret.owner;
|
||||
inherit (secret) mode group;
|
||||
text = builtins.hashString "md5" name;
|
||||
};
|
||||
}) config.age.secrets;
|
||||
}
|
||||
|
|
@ -53,6 +53,8 @@ super: rec {
|
|||
|
||||
forgejo = patch super.forgejo "patches/base/forgejo";
|
||||
|
||||
garage = patch super.garage_0_8 "patches/base/garage";
|
||||
|
||||
jellyfin = patch (super.jellyfin.override {
|
||||
ffmpeg = super.ffmpeg.override {
|
||||
withMfx = true;
|
||||
|
|
|
|||
264
patches/base/garage/print-chill-pills.patch
Normal file
264
patches/base/garage/print-chill-pills.patch
Normal file
|
|
@ -0,0 +1,264 @@
|
|||
diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs
|
||||
index bbde204..7bed4b0 100644
|
||||
--- a/src/db/bin/convert.rs
|
||||
+++ b/src/db/bin/convert.rs
|
||||
@@ -4,6 +4,28 @@ use garage_db::*;
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
+macro_rules! print {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ write!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
/// K2V command line interface
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(author, version, about, long_about = None)]
|
||||
diff --git a/src/db/lib.rs b/src/db/lib.rs
|
||||
index 11cae4e..ffef3fa 100644
|
||||
--- a/src/db/lib.rs
|
||||
+++ b/src/db/lib.rs
|
||||
@@ -25,6 +25,18 @@ use std::sync::Arc;
|
||||
|
||||
use err_derive::Error;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
#[derive(Clone)]
|
||||
pub struct Db(pub(crate) Arc<dyn IDb>);
|
||||
|
||||
diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs
|
||||
index 0d73588..6bf4ecc 100644
|
||||
--- a/src/garage/cli/cmd.rs
|
||||
+++ b/src/garage/cli/cmd.rs
|
||||
@@ -13,6 +13,28 @@ use garage_model::helper::error::Error as HelperError;
|
||||
use crate::admin::*;
|
||||
use crate::cli::*;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
+macro_rules! print {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ write!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
pub async fn cli_command_dispatch(
|
||||
cmd: Command,
|
||||
system_rpc_endpoint: &Endpoint<SystemRpc, ()>,
|
||||
diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs
|
||||
index 20813f1..f4baea2 100644
|
||||
--- a/src/garage/cli/init.rs
|
||||
+++ b/src/garage/cli/init.rs
|
||||
@@ -2,6 +2,18 @@ use std::path::PathBuf;
|
||||
|
||||
use garage_util::error::*;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
pub const READ_KEY_ERROR: &str = "Unable to read node key. It will be generated by your garage node the first time is it launched. Ensure that your garage node is currently running. (The node key is supposed to be stored in your metadata directory.)";
|
||||
|
||||
pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
|
||||
diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs
|
||||
index 3884bb9..ef55a66 100644
|
||||
--- a/src/garage/cli/layout.rs
|
||||
+++ b/src/garage/cli/layout.rs
|
||||
@@ -8,6 +8,28 @@ use garage_rpc::*;
|
||||
|
||||
use crate::cli::*;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
+macro_rules! print {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ write!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
pub async fn cli_layout_command_dispatch(
|
||||
cmd: LayoutOperation,
|
||||
system_rpc_endpoint: &Endpoint<SystemRpc, ()>,
|
||||
diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs
|
||||
index 2c6be2f..db6f25d 100644
|
||||
--- a/src/garage/cli/util.rs
|
||||
+++ b/src/garage/cli/util.rs
|
||||
@@ -17,6 +17,28 @@ use garage_model::s3::version_table::Version;
|
||||
|
||||
use crate::cli::structs::WorkerListOpt;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
+macro_rules! print {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ write!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
pub fn print_bucket_list(bl: Vec<Bucket>) {
|
||||
println!("List of buckets:");
|
||||
|
||||
diff --git a/src/k2v-client/bin/k2v-cli.rs b/src/k2v-client/bin/k2v-cli.rs
|
||||
index cdd63cc..dfa4df4 100644
|
||||
--- a/src/k2v-client/bin/k2v-cli.rs
|
||||
+++ b/src/k2v-client/bin/k2v-cli.rs
|
||||
@@ -11,6 +11,28 @@ use rusoto_core::Region;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
+macro_rules! print {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ write!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
/// K2V command line interface
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(author, version, about, long_about = None)]
|
||||
diff --git a/src/rpc/layout.rs b/src/rpc/layout.rs
|
||||
index 1030e3a..47eca49 100644
|
||||
--- a/src/rpc/layout.rs
|
||||
+++ b/src/rpc/layout.rs
|
||||
@@ -10,6 +10,28 @@ use garage_util::error::*;
|
||||
|
||||
use crate::ring::*;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! println {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ writeln!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
+macro_rules! print {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ write!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
/// The layout of the cluster, i.e. the list of roles
|
||||
/// which are assigned to each cluster node
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
diff --git a/src/util/formater.rs b/src/util/formater.rs
|
||||
index 2ea53eb..cc7d8a4 100644
|
||||
--- a/src/util/formater.rs
|
||||
+++ b/src/util/formater.rs
|
||||
@@ -1,3 +1,15 @@
|
||||
+use std::io::Write;
|
||||
+
|
||||
+macro_rules! print {
|
||||
+ () => (print!("\n"));
|
||||
+ ($fmt:expr) => ({
|
||||
+ write!(std::io::stdout(), $fmt).unwrap_or(())
|
||||
+ });
|
||||
+ ($fmt:expr, $($arg:tt)*) => ({
|
||||
+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(())
|
||||
+ })
|
||||
+}
|
||||
+
|
||||
pub fn format_table_to_string(data: Vec<String>) -> String {
|
||||
let data = data
|
||||
.iter()
|
||||
|
|
@ -44,7 +44,8 @@ in with hosts;
|
|||
"cluster/services/patroni/passwords/superuser.age".publicKeys = max ++ map systemKeys [ thunderskin VEGAS prophet ];
|
||||
"cluster/services/storage/secrets/heresy-encryption-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];
|
||||
"cluster/services/storage/secrets/external-storage-encryption-key-prophet.age".publicKeys = max ++ map systemKeys [ prophet ];
|
||||
"cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
|
||||
"cluster/services/storage/secrets/garage-rpc-secret.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
|
||||
"cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
|
||||
"cluster/services/wireguard/mesh-keys/checkmate.age".publicKeys = max ++ map systemKeys [ checkmate ];
|
||||
"cluster/services/wireguard/mesh-keys/thunderskin.age".publicKeys = max ++ map systemKeys [ thunderskin ];
|
||||
"cluster/services/wireguard/mesh-keys/VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ];
|
||||
|
|
|
|||
Loading…
Reference in a new issue