Compare commits

...

52 commits

Author SHA1 Message Date
Max
00f233e8a5 cluster/services/frangiclave: funny 2024-08-12 03:04:14 +02:00
Max
e2fe73039c cluster/services/frangiclave: test in simulacrum WIP 2024-08-12 03:04:14 +02:00
Max
e4f09218d5 cluster/services/frangiclave: retry_join 2024-08-12 03:04:14 +02:00
Max
ebc9b88c8d cluster/services/frangiclave: some cluster stuff 2024-08-12 03:04:14 +02:00
Max
ea3414c427 cluster/services/frangiclave: init trivial WIP 2024-08-12 03:04:14 +02:00
Max
6eee030b7a cluster/services/storage: register existing keys and buckets in incandescence 2024-08-12 03:04:03 +02:00
Max
75cebf4ed6 cluster/services/incandescence: add base layout for ascensions 2024-08-12 03:04:03 +02:00
Max
bc3cd82731 cluster/services/consul: implement runConsul incantation 2024-08-12 03:04:03 +02:00
Max
9cdf964c6c cluster/services/forge: define db 2024-08-12 03:04:03 +02:00
Max
bb22fe0176 cluster/services/patroni: implement raw format for locksmith provider 2024-08-12 03:04:03 +02:00
Max
d1f2bc1227 cluster/services/storage: define snakeoil passphrase for heresy, ensure encryption 2024-08-12 03:04:03 +02:00
Max
a48ee00f3d cluster/services/ways: add simulacrum deps 2024-08-12 03:04:03 +02:00
Max
9ed3655ccf cluster/services/storage: use recursive simulacrum deps 2024-08-12 03:04:03 +02:00
Max
945698a3ea cluster/services/patroni: add simulacrum deps 2024-08-12 03:04:03 +02:00
Max
f75c7b8522 cluster/services/locksmith: add simulacrum deps 2024-08-12 03:04:03 +02:00
Max
b258bab23e cluster/services/incandescence: add simulacrum deps 2024-08-12 03:04:03 +02:00
Max
e2296eace7 cluster/services/chant: add simulacrum deps 2024-08-12 03:04:03 +02:00
Max
304ae6e53c cluster/simulacrum: recursive service deps 2024-08-12 03:04:03 +02:00
Max
f322208f66 cluster/services/acme-client: implement augment for external ACME services 2024-08-12 03:04:03 +02:00
Max
7c4615ecfb cluster/simulacrum: implement nowhere, fix networking 2024-08-12 03:04:03 +02:00
Max
ec38e10fa9 cluster/services/forge: use forService 2024-08-12 03:04:03 +02:00
Max
5d9ff62afe cluster/services/dns: use patroni incandescence 2024-08-12 03:04:03 +02:00
Max
6d78b69601 cluster/services/patroni: implement incandescence provider for databases and users 2024-08-12 03:04:03 +02:00
Max
7129d44078 cluster/services/locksmith: only run secret generation command once 2024-08-12 03:04:03 +02:00
Max
76d205d114 cluster/services/locksmith: support skipping secret updates 2024-08-12 03:04:03 +02:00
Max
c8c9a6fbce modules/external-storage: implement detectFs for s3c4 2024-08-12 03:04:03 +02:00
Max
a1cad2efcd cluster/services/storage: use locksmith secrets for external storage 2024-08-12 03:04:03 +02:00
Max
c7f4e59908 cluster/services/storage: adjust test 2024-08-12 03:04:03 +02:00
Max
baed1ce871 cluster/services/storage: use incandescence 2024-08-12 03:04:03 +02:00
Max
014c1f9cd2 cluster/services/incandescence: init 2024-08-12 03:04:03 +02:00
Max
34704c8f08 modules/external-storage: support locksmith secrets 2024-08-12 03:04:03 +02:00
Max
ccc2a47880 cluster/services/storage: implement s3ql key format 2024-08-12 03:04:03 +02:00
Max
05cd729e90 cluster/services/hercules-ci-multi-agent: use forService 2024-08-12 03:04:03 +02:00
Max
8d0a2f00cc cluster/services/monitoring: use forService 2024-08-12 03:04:03 +02:00
Max
ff26e1ebc1 checks/garage: drop 2024-08-12 03:04:03 +02:00
Max
b848084dd8 packages/catalog: expose simulacrum checks differently 2024-08-12 03:04:03 +02:00
Max
fe8ddd4094 cluster/simulacrum: expose checks 2024-08-12 03:04:03 +02:00
Max
030b680b33 cluster/services/forge: use forService 2024-08-12 03:04:03 +02:00
Max
b453b0bb21 cluster/services/attic: use forService 2024-08-12 03:04:03 +02:00
Max
b6e0390555 cluster/lib: implement config.lib.forService for better option filtering 2024-08-12 03:04:03 +02:00
Max
bbe3373c2e cluster/simulacrum: set testConfig 2024-08-12 03:04:03 +02:00
Max
0ed4870b65 cluster/lib: introduce testConfig 2024-08-12 03:04:03 +02:00
Max
8ec13f5c87 cluster/services/storage: test in simulacrum 2024-08-12 03:04:03 +02:00
Max
5d52f72940 cluster/services/consul: test in simulacrum 2024-08-12 03:04:03 +02:00
Max
1af67b80ed cluster/services/wireguard: make simulacrum compatible 2024-08-12 03:04:03 +02:00
Max
a810717843 cluster/catalog: support snakeoil secrets 2024-08-12 03:04:03 +02:00
Max
bd39fc5d07 cluster/simulacrum: init 2024-08-12 03:04:03 +02:00
Max
25c001c182 cluster/lib: implement simulacrum options 2024-08-12 03:04:03 +02:00
Max
d944dee3bc WIP ENABLE DEBUG MODE 2024-08-12 02:56:57 +02:00
Max
aac5163a8b cluster/lib: implement injectNixosConfigForServices to select individual services 2024-08-12 02:56:57 +02:00
Max
01c74f62cf checks: add fake external storage module 2024-08-12 02:56:57 +02:00
Max
0110a4a0c3 checks: add a bunch of snakeoil keys 2024-08-12 02:56:57 +02:00
67 changed files with 1333 additions and 358 deletions

View file

@ -46,6 +46,7 @@ in
};
}) // (if secretConfig.shared then let
secretFile = "${svcName}-${secretName}.age";
snakeoilFile = "${svcName}-${secretName}-snakeoil.txt";
in {
editSecret = {
description = "Edit this secret";
@ -54,15 +55,31 @@ in
agenix -e '${secretFile}'
'';
};
} else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
secretFile = "${svcName}-${secretName}-${node}.age";
in {
description = "Edit this secret for '${node}'";
command = ''
${setupCommands secretFile [ node ]}
agenix -e '${secretFile}'
'';
})));
editSnakeoil = {
description = "Edit this secret's snakeoil";
command = ''
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
'';
};
} else lib.mkMerge [
(lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
secretFile = "${svcName}-${secretName}-${node}.age";
in {
description = "Edit this secret for '${node}'";
command = ''
${setupCommands secretFile [ node ]}
agenix -e '${secretFile}'
'';
})))
(lib.mapAttrs' (name: lib.nameValuePair "editSnakeoilInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
snakeoilFile = "${svcName}-${secretName}-${node}-snakeoil.txt";
in {
description = "Edit this secret's snakeoil for '${node}'";
command = ''
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
'';
})))
]);
};
}) svcConfig.secrets))
lib.concatLists

View file

@ -16,6 +16,8 @@ lib.evalModules {
./lib/port-magic-multi.nix
./lib/mesh.nix
./lib/secrets.nix
./lib/testing.nix
./lib/lib.nix
./import-services.nix
];

View file

@ -2,9 +2,9 @@
with lib;
{
options.out.injectNixosConfig = mkOption {
description = "NixOS configuration to inject into the given host.";
type = with types; functionTo raw;
options.out = mkOption {
description = "Output functions.";
type = with types; lazyAttrsOf (functionTo raw);
default = const [];
};
}

12
cluster/lib/lib.nix Normal file
View file

@ -0,0 +1,12 @@
{ config, lib, ... }:
{
options.lib = {
forService = lib.mkOption {
description = "Enable these definitions for a particular service only.";
type = lib.types.functionTo lib.types.raw;
readOnly = true;
default = service: lib.mkIf (!config.simulacrum || lib.any (s: s == service) config.testConfig.activeServices);
};
};
}

View file

@ -52,6 +52,24 @@ in
}));
default = {};
};
simulacrum = {
enable = mkEnableOption "testing this service in the Simulacrum";
deps = mkOption {
description = "Other services to include.";
type = with types; listOf str;
default = [];
};
settings = mkOption {
description = "NixOS test configuration.";
type = types.deferredModule;
default = {};
};
augments = mkOption {
description = "Cluster augments (will be propagated).";
type = types.deferredModule;
default = {};
};
};
};
config.otherNodes = builtins.mapAttrs (const filterGroup) config.nodes;
}

View file

@ -39,7 +39,11 @@ in
default = {};
};
config.out.injectNixosConfig = hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) config.services)) ++ [
introspectionModule
];
config.out = {
injectNixosConfigForServices = services: hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) (lib.getAttrs services config.services))) ++ [
introspectionModule
];
injectNixosConfig = config.out.injectNixosConfigForServices (lib.attrNames config.services);
};
}

15
cluster/lib/testing.nix Normal file
View file

@ -0,0 +1,15 @@
{ lib, ... }:
{
options = {
simulacrum = lib.mkOption {
description = "Whether we are in the Simulacrum.";
type = lib.types.bool;
default = false;
};
testConfig = lib.mkOption {
type = lib.types.attrs;
readOnly = true;
};
};
}

View file

@ -3,6 +3,7 @@
{
imports = [
./catalog
./simulacrum/checks.nix
];
options.cluster = lib.mkOption {

View file

@ -1,13 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
-> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
-> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
-> Ovax-grease ^1$]}H G4 FpDF XKHkj{
IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
--- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
ÜA³×oÈð:±­‹`ÜVd±å(Kät:fk¼}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69 il`ÛÆJKwAè8­y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ýw Ê
ÆÜßÌZî!^þRˆéÿv­¾…ïkÊp»ÛPÌ)ý̆ÍpÓV5²F΄ÆÚÙÚÞhBÇ»ß b# Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè}€ß ƒ¬E}~ZHûjmyq{òxŠÉôß<C3B4>"”éÀ´C#šójÿÐ.ò§y Ô£¸<0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që

View file

@ -0,0 +1,60 @@
{ config, pkgs, ... }:
let
lift = config;
in
{
nowhere.names = {
"acme-v02.api.letsencrypt.org" = "stepCa";
"api.buypass.com" = "stepCa";
};
nodes.nowhere = { config, ... }: {
links.stepCa.protocol = "https";
environment.etc.step-ca-password.text = "";
services = {
step-ca = {
enable = true;
address = config.links.stepCa.ipv4;
inherit (config.links.stepCa) port;
intermediatePasswordFile = "/etc/step-ca-password";
settings = {
root = "${lift.nowhere.certs.ca}/ca.pem";
crt = "${lift.nowhere.certs.intermediate}/cert.pem";
key = "${lift.nowhere.certs.intermediate}/cert-key.pem";
address = config.links.stepCa.tuple;
db = {
type = "badgerv2";
dataSource = "/var/lib/step-ca/db";
};
authority.provisioners = [
{
type = "ACME";
name = "snakeoil";
challenges = [
"dns-01"
"http-01"
];
}
];
};
};
nginx.virtualHosts = {
"acme-v02.api.letsencrypt.org".locations."/".extraConfig = ''
rewrite /directory /acme/snakeoil/directory break;
'';
"api.buypass.com".locations."/".extraConfig = ''
rewrite /acme/directory /acme/snakeoil/directory break;
'';
};
};
};
defaults.environment.etc."dummy-secrets/acmeDnsApiKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
defaults.environment.etc."dummy-secrets/acmeDnsDirectKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
defaults.environment.etc."dummy-secrets/acmeDnsDbCredentials".text = "PGPASSWORD=simulacrum";
}

View file

@ -2,5 +2,6 @@
services.acme-client = {
nodes.client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
nixos.client = ./client.nix;
simulacrum.augments = ./augment.nix;
};
}

View file

@ -33,7 +33,7 @@
};
};
garage = {
garage = config.lib.forService "attic" {
keys.attic.locksmith = {
nodes = config.services.attic.nodes.server;
owner = "atticd";
@ -48,7 +48,7 @@
serverAddrs = map
(node: depot.hours.${node}.interfaces.primary.addrPublic)
config.services.attic.nodes.server;
in {
in config.lib.forService "attic" {
cache.target = serverAddrs;
};

View file

@ -6,5 +6,6 @@
nixos.listener = [
./listener.nix
];
simulacrum.deps = [ "consul" ];
};
}

View file

@ -10,6 +10,13 @@ let
in
{
system.extraIncantations = {
runConsul = i: script: i.execShellWith [ config.services.consul.package ] ''
export CONSUL_HTTP_ADDR='${config.links.consulAgent.tuple}'
${script}
'';
};
links.consulAgent.protocol = "http";
services.consul = {

View file

@ -22,6 +22,11 @@ in
];
ready = ./ready.nix;
};
simulacrum = {
enable = true;
deps = [ "wireguard" ];
settings = ./test.nix;
};
};
dns.records."consul-remote.internal".consulService = "consul-remote";

View file

@ -0,0 +1,19 @@
{
testScript = ''
import json
start_all()
with subtest("should form cluster"):
for machine in machines:
machine.succeed("systemctl start consul-ready.service")
for machine in machines:
consulConfig = json.loads(machine.succeed("cat /etc/consul.json"))
addr = consulConfig["addresses"]["http"]
port = consulConfig["ports"]["http"]
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
memberList = machine.succeed(f"{setEnv} consul members --status=alive")
for machine2 in machines:
assert machine2.name in memberList
'';
}

View file

@ -43,9 +43,6 @@ in {
links.localAuthoritativeDNS = {};
age.secrets = {
acmeDnsDbCredentials = {
file = ./acme-dns-db-credentials.age;
};
acmeDnsDirectKey = {
file = ./acme-dns-direct-key.age;
};
@ -78,8 +75,12 @@ in {
};
};
services.locksmith.waitForSecrets.acme-dns = [
"patroni-acmedns"
];
systemd.services.acme-dns.serviceConfig.EnvironmentFile = with config.age.secrets; [
acmeDnsDbCredentials.path
"/run/locksmith/patroni-acmedns"
acmeDnsDirectKey.path
];

View file

@ -58,6 +58,16 @@ in
};
};
patroni = {
databases.acmedns = {};
users.acmedns = {
locksmith = {
nodes = config.services.dns.nodes.authoritative;
format = "envFile";
};
};
};
dns.records = {
securedns.consulService = "securedns";
"acme-dns-challenge.internal".consulService = "acme-dns";

View file

@ -13,25 +13,36 @@
nodes = server;
owner = "forgejo";
};
dbCredentials.nodes = server;
};
};
ways.forge.target = let
ways.forge = let
host = builtins.head config.services.forge.nodes.server;
in config.hostLinks.${host}.forge.url;
in config.lib.forService "forge" {
target = config.hostLinks.${host}.forge.url;
};
garage = {
patroni = config.lib.forService "forge" {
databases.forge = {};
users.forge.locksmith = {
nodes = config.services.forge.nodes.server;
format = "raw";
};
};
garage = config.lib.forService "forge" {
keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
buckets.forgejo.allow.forgejo = [ "read" "write" ];
};
monitoring.blackbox.targets.forge = {
monitoring.blackbox.targets.forge = config.lib.forService "forge" {
address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
module = "https2xx";
};
dns.records."ssh.forge".target = map
(node: depot.hours.${node}.interfaces.primary.addrPublic)
config.services.forge.nodes.server;
dns.records = config.lib.forService "forge" {
"ssh.forge".target = map
(node: depot.hours.${node}.interfaces.primary.addrPublic)
config.services.forge.nodes.server;
};
}

View file

@ -26,6 +26,7 @@ in
services.locksmith.waitForSecrets.forgejo = [
"garage-forgejo-id"
"garage-forgejo-secret"
"patroni-forge"
];
services.forgejo = {
@ -39,7 +40,7 @@ in
inherit (patroni) port;
name = "forge";
user = "forge";
passwordFile = secrets.dbCredentials.path;
passwordFile = "/run/locksmith/patroni-forge";
};
settings = {
DEFAULT = {

View file

@ -0,0 +1,27 @@
{ config, ... }:
{
services.frangiclave = {
nodes = {
server = [ "VEGAS" "grail" "prophet" ];
cluster = config.services.frangiclave.nodes.server;
agent = []; # all nodes, for vault-agent, secret templates, etc.
};
meshLinks = {
server.link.protocol = "http";
cluster.link.protocol = "http";
};
nixos = {
server = [
./server.nix
];
cluster = [];
agent = [];
};
simulacrum = {
enable = true;
deps = [ "wireguard" "consul" ];
settings = ./test.nix;
};
};
}

View file

@ -0,0 +1,34 @@
{ cluster, config, depot, lib, ... }:
let
apiLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server;
clusterLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-cluster;
in
{
services.vault = {
enable = true;
package = depot.packages.openbao;
address = apiLink.tuple;
extraConfig = /*hcl*/ ''
api_addr = "${apiLink.url}"
cluster_addr = "${clusterLink.url}"
'';
storageBackend = "raft";
storageConfig = /*hcl*/ ''
node_id = "x${builtins.hashString "sha256" "frangiclave-node-${config.networking.hostName}"}"
${
lib.pipe (cluster.config.services.frangiclave.otherNodes.server config.networking.hostName) [
(map (node: cluster.config.hostLinks.${node}.frangiclave-server))
(map (link: /*hcl*/ ''
retry_join {
leader_api_addr = "${link.url}"
}
''))
(lib.concatStringsSep "\n")
]
}
'';
};
}

View file

@ -0,0 +1,12 @@
{ lib, ... }:
{
interactive.defaults = { cluster, config, ... }: {
config = lib.mkIf config.services.vault.enable {
environment.variables.VAULT_ADDR = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server.url;
environment.systemPackages = [ config.services.vault.package ];
};
};
testScript = "assert False";
}

View file

@ -62,7 +62,7 @@
lib.unique
(map (x: "hci-agent-${x}"))
];
in {
in config.lib.forService "hercules-ci-multi-agent" {
keys = lib.genAttrs hciAgentKeys (lib.const {});
buckets.nix-store = {
allow = lib.genAttrs hciAgentKeys (lib.const [ "read" "write" ]);

View file

@ -0,0 +1,20 @@
{ config, ... }:
{
imports = [
./options.nix
];
services.incandescence = {
nodes = {
provider = config.services.consul.nodes.agent;
};
nixos = {
provider = [
./provider.nix
./provider-options.nix
];
};
simulacrum.deps = [ "consul" ];
};
}

View file

@ -0,0 +1,22 @@
{ lib, ... }:
let
inherit (lib) mkOption;
inherit (lib.types) attrsOf listOf submodule str;
in
{
options.incandescence = {
providers = mkOption {
type = attrsOf (submodule ({ name, ... }: {
options = {
objects = mkOption {
type = attrsOf (listOf str);
default = { };
};
};
}));
default = { };
};
};
}

View file

@ -0,0 +1,72 @@
{ lib, ... }:
let
inherit (lib) mkEnableOption mkOption;
inherit (lib.types) attrsOf functionTo ints listOf nullOr package submodule str;
in
{
options.services.incandescence = {
providers = mkOption {
type = attrsOf (submodule ({ name, ... }: {
options = {
locksmith = mkEnableOption "Locksmith integration";
wantedBy = mkOption {
type = listOf str;
};
partOf = mkOption {
type = listOf str;
};
wants = mkOption {
type = listOf str;
default = [ ];
};
after = mkOption {
type = listOf str;
default = [ ];
};
packages = mkOption {
type = listOf package;
default = [ ];
};
formulae = mkOption {
type = attrsOf (submodule ({ ... }: {
options = {
deps = mkOption {
type = listOf str;
default = [ ];
};
create = mkOption {
type = functionTo str;
};
change = mkOption {
type = nullOr (functionTo str);
default = null;
};
destroy = mkOption {
type = str;
};
destroyAfterDays = mkOption {
type = ints.unsigned;
default = 0;
};
};
}));
default = { };
};
};
}));
default = { };
};
};
}

View file

@ -0,0 +1,138 @@
{ cluster, config, lib, ... }:
let
inherit (lib) concatStringsSep escapeShellArg flatten filter filterAttrs length mapAttrs mapAttrs' mapAttrsToList mkIf mkMerge pipe stringToCharacters;
cfg = config.services.incandescence;
clusterCfg = cluster.config.incandescence;
in
{
systemd.services = pipe cfg.providers [
(mapAttrsToList (provider: providerConfig: pipe providerConfig.formulae [
(mapAttrsToList (formula: formulaConfig: let
kvRoot = "services/incandescence/providers/${provider}/formulae/${formula}";
time = "$(date +%s)";
in {
"ignite-${provider}-${formula}-create" = {
description = "Ignite Creation: ${provider} - ${formula}";
wantedBy = [ "incandescence-${provider}.target" ];
before = [ "incandescence-${provider}.target" ];
wants = providerConfig.wants ++ map (dep: "ignite-${provider}-${dep}-create.service") formulaConfig.deps;
after = providerConfig.after ++ map (dep: "ignite-${provider}-${dep}-create.service") formulaConfig.deps;
serviceConfig.Type = "oneshot";
distributed.enable = true;
path = [ config.services.consul.package ] ++ providerConfig.packages;
script = pipe clusterCfg.providers.${provider}.objects.${formula} [
(map (object: ''
if ! consul kv get ${kvRoot}/${object}/alive >/dev/null; then
echo "Create ${formula}: ${object}"
if (
${formulaConfig.create object}
)
then
consul kv put ${kvRoot}/${object}/alive true
consul kv delete ${kvRoot}/${object}/destroyOn
else
echo "Creation failed: ${object}"
fi
fi
''))
(concatStringsSep "\n")
];
};
"ignite-${provider}-${formula}-change" = mkIf (formulaConfig.change != null) {
description = "Ignite Change: ${provider} - ${formula}";
wantedBy = [ "incandescence-${provider}.target" ];
before = [ "incandescence-${provider}.target" ];
wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-create.service" ] ++ map (dep: "ignite-${provider}-${dep}-change.service") formulaConfig.deps;
after = providerConfig.after ++ [ "ignite-${provider}-${formula}-create.service" ] ++ map (dep: "ignite-${provider}-${dep}-change.service") formulaConfig.deps;
serviceConfig.Type = "oneshot";
distributed.enable = true;
path = [ config.services.consul.package ] ++ providerConfig.packages;
script = pipe clusterCfg.providers.${provider}.objects.${formula} [
(map (object: ''
echo "Change ${formula}: ${object}"
(
${formulaConfig.change object}
) || echo "Change failed: ${object}"
''))
(concatStringsSep "\n")
];
};
"ignite-${provider}-${formula}-destroy" = {
description = "Ignite Destruction: ${provider} - ${formula}";
wantedBy = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
before = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-change.service" ];
after = providerConfig.after ++ [ "ignite-${provider}-${formula}-change.service" ];
serviceConfig.Type = "oneshot";
distributed.enable = true;
path = [ config.services.consul.package ] ++ providerConfig.packages;
script = let
fieldNum = pipe kvRoot [
stringToCharacters
(filter (x: x == "/"))
length
(builtins.add 2)
toString
];
keyFilter = pipe clusterCfg.providers.${provider}.objects.${formula} [
(map (x: escapeShellArg "^${x}$"))
(concatStringsSep " \\\n -e ")
];
destroyAfterDays = toString formulaConfig.destroyAfterDays;
in ''
consul kv get --keys ${kvRoot}/ | cut -d/ -f${fieldNum} | grep -v -e ${keyFilter} | while read object; do
if consul kv get ${kvRoot}/$object/alive >/dev/null; then
destroyOn="$(consul kv get ${kvRoot}/$object/destroyOn || true)"
if [[ -z "$destroyOn" && "${destroyAfterDays}" -ne 0 ]]; then
echo "Schedule ${formula} for destruction in ${destroyAfterDays} days: $object"
consul kv put ${kvRoot}/$object/destroyOn "$((${time} + 86400 * ${destroyAfterDays}))"
elif [[ "${destroyAfterDays}" -eq 0 || "${time}" -ge "$destroyOn" ]]; then
echo "Destroy ${formula}: $object"
export OBJECT="$object"
if (
${formulaConfig.destroy}
)
then
consul kv delete --recurse ${kvRoot}/$object
else
echo "Destruction failed: $object"
fi
else
echo "Scheduled for destruction on $destroyOn (now: ${time})"
fi
fi
done
'';
};
}))
]))
flatten
mkMerge
];
systemd.targets = mapAttrs' (provider: providerConfig: {
name = "incandescence-${provider}";
value = {
description = "An Incandescence | ${provider}";
inherit (providerConfig) wantedBy partOf;
};
}) cfg.providers;
services.locksmith.providers = mapAttrs (provider: providerConfig: {
wantedBy = [ "incandescence-${provider}.target" ];
after = [ "incandescence-${provider}.target" ];
}) (filterAttrs (_: providerConfig: providerConfig.locksmith) cfg.providers);
system.ascensions = mapAttrs' (provider: providerConfig: {
name = "incandescence-${provider}";
value = {
distributed = true;
requiredBy = map (formula: "ignite-${provider}-${formula}-create.service") (lib.attrNames providerConfig.formulae);
before = map (formula: "ignite-${provider}-${formula}-create.service") (lib.attrNames providerConfig.formulae);
incantations = lib.mkDefault (i: []);
};
}) cfg.providers;
}

View file

@ -14,5 +14,6 @@
./provider.nix
];
};
simulacrum.deps = [ "chant" "consul" ];
};
}

View file

@ -28,6 +28,10 @@ in
command = mkOption {
type = types.coercedTo types.package (package: "${package}") types.str;
};
checkUpdate = mkOption {
type = types.coercedTo types.package (package: "${package}") types.str;
default = "true";
};
owner = mkOption {
type = types.str;
default = "root";
@ -72,20 +76,27 @@ in
activeNodes = lib.unique (lib.flatten (lib.mapAttrsToList (_: secret: secret.nodes) activeSecrets));
secretNames = map (name: "${providerRoot}-${name}/") (lib.attrNames activeSecrets);
createSecret = { path, nodes, owner, mode, group, command }: ''
consul kv put ${lib.escapeShellArg path}/mode ${lib.escapeShellArg mode}
consul kv put ${lib.escapeShellArg path}/owner ${lib.escapeShellArg owner}
consul kv put ${lib.escapeShellArg path}/group ${lib.escapeShellArg group}
${lib.concatStringsSep "\n" (map (node: ''
consul kv put ${lib.escapeShellArg path}/recipient/${node} "$( (${command}) | age --encrypt --armor -r ${lib.escapeShellArg depot.hours.${node}.ssh.id.publicKey})"
'') nodes)}
createSecret = { path, nodes, owner, mode, group, command, checkUpdate }: ''
if (${checkUpdate}); then
consul kv put ${lib.escapeShellArg path}/mode ${lib.escapeShellArg mode}
consul kv put ${lib.escapeShellArg path}/owner ${lib.escapeShellArg owner}
consul kv put ${lib.escapeShellArg path}/group ${lib.escapeShellArg group}
secret="$(mktemp -ut)"
(${command}) > "$secret"
${lib.concatStringsSep "\n" (map (node: ''
consul kv put ${lib.escapeShellArg path}/recipient/${node} "$(age < "$secret" --encrypt --armor -r ${lib.escapeShellArg depot.hours.${node}.ssh.id.publicKey})"
'') nodes)}
else
echo Skipping update for ${lib.escapeShellArg path}
fi
'';
in ''
# create/update secrets
umask 77
${lib.pipe activeSecrets [
(lib.mapAttrsToList (secretName: secretConfig: createSecret {
path = "${providerRoot}-${secretName}";
inherit (secretConfig) nodes mode owner group command;
inherit (secretConfig) nodes mode owner group command checkUpdate;
}))
(lib.concatStringsSep "\n")
]}

View file

@ -72,7 +72,7 @@ in
};
};
garage = {
garage = config.lib.forService "monitoring" {
keys = {
loki-ingest.locksmith = {
nodes = config.services.monitoring.nodes.logging;

View file

@ -0,0 +1,91 @@
{ cluster, config, lib, pkgs, ... }:
let
inherit (cluster.config.services.patroni) secrets;
patroni = cluster.config.links.patroni-pg-access;
cfg = cluster.config.patroni;
writeQueryFile = pkgs.writeText "patroni-query.sql";
psqlRunFile = file: ''
export PGPASSWORD="$(< ${secrets.PATRONI_SUPERUSER_PASSWORD.path})"
while ! ${config.services.patroni.postgresqlPackage}/bin/psql 'host=${patroni.ipv4} port=${patroni.portStr} dbname=postgres user=postgres' --tuples-only --csv --file="${file}"; do
sleep 3
done
'';
psql = query: psqlRunFile (writeQueryFile query);
psqlSecret = getSecret: queryTemplate: let
queryTemplateFile = writeQueryFile queryTemplate;
in ''
umask 77
secretFile="$(mktemp -ut patroniSecret.XXXXXXXXXXXXXXXX)"
queryFile="$(mktemp -ut patroniQuery.XXXXXXXXXXXXXXXX)"
trap "rm -f $secretFile $queryFile" EXIT
${getSecret} > "$secretFile"
cp --no-preserve=mode ${queryTemplateFile} "$queryFile"
${pkgs.replace-secret}/bin/replace-secret '@SECRET@' "$secretFile" "$queryFile"
${psqlRunFile "$queryFile"}
'';
genPassword = pkgs.writeShellScript "patroni-generate-user-password" ''
umask 77
base64 -w0 /dev/urandom | tr -d /+ | head -c256 | tee "/run/keys/locksmith-provider-patroni-$1"
'';
in
{
services.incandescence.providers.patroni = lib.mkIf config.services.haproxy.enable {
locksmith = true;
wantedBy = [ "patroni.service" "multi-user.target" ];
partOf = [ "patroni.service" ];
wants = [ "postgresql.service" ];
after = [ "postgresql.service" ];
formulae = {
user = {
destroyAfterDays = 0;
create = user: psqlSecret "${genPassword} ${user}" ''
CREATE USER ${user} PASSWORD '@SECRET@';
'';
destroy = psqlSecret "printenv OBJECT" ''
DROP USER @SECRET@;
'';
};
database = {
destroyAfterDays = 30;
deps = [ "user" ];
create = db: psql ''
CREATE DATABASE ${db} OWNER ${cfg.databases.${db}.owner};
'';
destroy = psqlSecret "printenv OBJECT" ''
DROP DATABASE @SECRET@;
'';
};
};
};
services.locksmith.providers.patroni = lib.mkIf config.services.haproxy.enable {
secrets = lib.mapAttrs (user: userConfig: {
command = {
envFile = ''
echo "PGPASSWORD=$(cat /run/keys/locksmith-provider-patroni-${user})"
rm -f /run/keys/locksmith-provider-patroni-${user}
'';
pgpass = ''
echo "*:*:*:${user}:$(cat /run/keys/locksmith-provider-patroni-${user})"
rm -f /run/keys/locksmith-provider-patroni-${user}
'';
raw = ''
cat /run/keys/locksmith-provider-patroni-${user}
rm -f /run/keys/locksmith-provider-patroni-${user}
'';
}.${userConfig.locksmith.format};
checkUpdate = "test -e /run/keys/locksmith-provider-patroni-${user}";
inherit (userConfig.locksmith) nodes;
}) cfg.users;
};
}

View file

@ -1,6 +1,11 @@
{ config, lib, ... }:
{ config, ... }:
{
imports = [
./options.nix
./incandescence.nix
];
links = {
patroni-pg-internal.ipv4 = "0.0.0.0";
patroni-api.ipv4 = "0.0.0.0";
@ -15,6 +20,7 @@
worker = [
./worker.nix
./metrics.nix
./create-databases.nix
];
haproxy = ./haproxy.nix;
};
@ -30,5 +36,6 @@
PATRONI_REWIND_PASSWORD = default;
metricsCredentials.nodes = nodes.worker;
};
simulacrum.deps = [ "consul" "incandescence" "locksmith" ];
};
}

View file

@ -0,0 +1,10 @@
{ config, lib, ... }:
{
incandescence.providers.patroni = {
objects = {
user = lib.attrNames config.patroni.users;
database = lib.attrNames config.patroni.databases;
};
};
}

View file

@ -0,0 +1,37 @@
{ lib, ... }:
let
inherit (lib) mkOption;
inherit (lib.types) attrsOf enum listOf submodule str;
in
{
options.patroni = {
databases = mkOption {
type = attrsOf (submodule ({ name, ... }: {
options = {
owner = mkOption {
type = str;
default = name;
};
};
}));
};
users = mkOption {
type = attrsOf (submodule ({ ... }: {
options = {
locksmith = {
nodes = mkOption {
type = listOf str;
default = [];
};
format = mkOption {
type = enum [ "pgpass" "envFile" "raw" ];
default = "pgpass";
};
};
};
}));
};
};
}

View file

@ -7,6 +7,8 @@ in
{
imports = [
./options.nix
./incandescence.nix
./simulacrum/test-data.nix
];
services.storage = {
@ -30,11 +32,16 @@ in
heresy = [
./heresy.nix
./s3ql-upgrades.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-heresy-passphrase.nix
];
garage = [
./garage.nix
./garage-options.nix
./garage-layout.nix
./incandescence-ascensions.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-rpc-secret.nix
];
garageConfig = [
./garage-gateway.nix
@ -48,6 +55,11 @@ in
garageInternal = [ ./garage-internal.nix ];
garageExternal = [ ./garage-external.nix ];
};
simulacrum = {
enable = true;
deps = [ "consul" "locksmith" "incandescence" "patroni" "ways" ];
settings = ./test.nix;
};
};
links = {
@ -86,7 +98,10 @@ in
};
garage = {
keys.storage-prophet = {};
keys.storage-prophet.locksmith = {
nodes = [ "prophet" ];
format = "s3ql";
};
buckets.storage-prophet = {
allow.storage-prophet = [ "read" "write" ];
};

View file

@ -8,7 +8,7 @@ in
services.external-storage = {
fileSystems.external = {
mountpoint = "/srv/storage";
authFile = ./secrets/external-storage-auth-${hostName}.age;
locksmithSecret = "garage-storage-${hostName}";
backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
backendOptions = [ "disable-expect100" ];
};

View file

@ -26,62 +26,6 @@ let
sleep 1
done
}
# FIXME: returns bogus empty string when one of the lists is empty
diffAdded() {
comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
}
diffRemoved() {
comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
}
# FIXME: this does not handle list items with spaces
listKeys() {
garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
}
ensureKeys() {
old="$(listKeys)"
if [[ -z "$1" ]]; then
for key in $old; do
garage key delete --yes "$key"
done
elif [[ -z "$old" ]]; then
for key in $1; do
# don't print secret key
garage key new --name "$key" >/dev/null
echo Key "$key" was created.
done
else
diffAdded "$old" "$1" | while read key; do
# don't print secret key
garage key new --name "$key" >/dev/null
echo Key "$key" was created.
done
diffRemoved "$old" "$1" | while read key; do
garage key delete --yes "$key"
done
fi
}
listBuckets() {
garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
}
ensureBuckets() {
old="$(listBuckets)"
if [[ -z "$1" ]]; then
for bucket in $old; do
garage bucket delete --yes "$bucket"
done
elif [[ -z "$old" ]]; then
for bucket in $1; do
garage bucket create "$bucket"
done
else
diffAdded "$old" "$1" | while read bucket; do
garage bucket create "$bucket"
done
diffRemoved "$old" "$1" | while read bucket; do
garage bucket delete --yes "$bucket"
done
fi
}
'';
in
@ -118,7 +62,7 @@ in
};
format = mkOption {
description = "Locksmith secret format.";
type = enum [ "files" "aws" "envFile" ];
type = enum [ "files" "aws" "envFile" "s3ql" ];
default = "files";
};
owner = mkOption {
@ -203,9 +147,7 @@ in
garage layout apply --version 1
'';
};
garage-apply = {
distributed.enable = true;
wantedBy = [ "garage.service" "multi-user.target" ];
garage-ready = {
wants = [ "garage.service" ];
after = [ "garage.service" "garage-layout-init.service" ];
path = [ config.services.garage.package ];
@ -219,54 +161,70 @@ in
script = ''
source ${garageShellLibrary}
waitForGarageOperational
ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
# key permissions
${lib.pipe cfg.keys [
(lib.mapAttrsToList (key: kCfg: ''
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
''))
(lib.concatStringsSep "\n")
]}
# bucket permissions
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg:
lib.mapAttrsToList (key: perms: ''
garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
'') bCfg.allow
))
lib.flatten
(lib.concatStringsSep "\n")
]}
# bucket quotas
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg: ''
garage bucket set-quotas '${bucket}' \
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
''))
(lib.concatStringsSep "\n")
]}
# bucket website access
${lib.pipe cfg.buckets [
(lib.mapAttrsToList (bucket: bCfg: ''
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
''))
(lib.concatStringsSep "\n")
]}
'';
};
};
services.incandescence.providers.garage = {
locksmith = true;
wantedBy = [ "garage.service" "multi-user.target" ];
partOf = [ "garage.service" ];
wants = [ "garage-ready.service" ];
after = [ "garage-ready.service" ];
packages = [
config.services.garage.package
];
formulae = {
key = {
destroyAfterDays = 0;
create = key: ''
# don't print secret key
garage key new --name ${lib.escapeShellArg key} >/dev/null
echo Key ${lib.escapeShellArg key} was created.
'';
destroy = ''
garage key delete --yes "$OBJECT"
'';
change = key: let
kCfg = cfg.keys.${key};
in ''
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
'';
};
bucket = {
deps = [ "key" ];
destroyAfterDays = 30;
create = bucket: ''
garage bucket create ${lib.escapeShellArg bucket}
'';
destroy = ''
garage bucket delete --yes "$OBJECT"
'';
change = bucket: let
bCfg = cfg.buckets.${bucket};
in ''
# permissions
${lib.concatStringsSep "\n" (lib.flatten (
lib.mapAttrsToList (key: perms: ''
garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
'') bCfg.allow
))}
# quotas
garage bucket set-quotas ${lib.escapeShellArg bucket} \
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
# website access
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
'';
};
};
};
services.locksmith.providers.garage = {
wantedBy = [ "garage-apply.service" ];
after = [ "garage-apply.service" ];
secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
common = {
inherit (kCfg.locksmith) mode owner group nodes;
@ -291,6 +249,12 @@ in
AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
'';
s3ql = ''
[s3c]
storage-url: s3c4://
backend-login: @@GARAGE_KEY_ID@@
backend-password: @@GARAGE_SECRET_KEY@@
'';
}.${kCfg.locksmith.format};
in {
${key} = common // {

View file

@ -63,6 +63,8 @@ in
};
systemd.services.garage = {
requires = [ "consul-ready.service" ];
after = [ "consul-ready.service" ];
unitConfig = {
RequiresMountsFor = [ cfg.settings.data_dir ];
};

View file

@ -11,6 +11,7 @@
unitDescription = "Heresy Filesystem";
authFile = ./secrets/heresy-encryption-key.age;
underlay = "heresy";
encrypt = true;
};
};
}

View file

@ -0,0 +1,18 @@
{ config, lib, ... }:
{
system.ascensions = {
incandescence-garage = lib.mkIf (config.services.incandescence.providers ? garage) {
incantations = i: [
(i.runGarage /*bash*/ ''
garage bucket list | tail -n +2 | cut -d' ' -f3 | while read bucket; do
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/bucket/$1/alive" true''} "$bucket"
done
garage key list | tail -n +2 | cut -d' ' -f5 | while read key; do
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/key/$1/alive" true''} "$key"
done
'')
];
};
};
}

View file

@ -0,0 +1,10 @@
{ config, lib, ... }:
{
incandescence.providers.garage = {
objects = {
key = lib.attrNames config.garage.keys;
bucket = lib.attrNames config.garage.buckets;
};
};
}

View file

@ -1,11 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
-> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
-> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
--- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
™ªg¬NÑ 8´¨\K!p «ï…7ù¶käõ¯#ŒÏuµ*{}Tþ0·|@Éÿà E>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
µÿÂ9gÏi"Q –ÿ
™›Ù®à

View file

@ -0,0 +1,8 @@
{
environment.etc."dummy-secrets/storageAuth-heresy".text = ''
[local]
storage-url: local://
fs-passphrase: simulacrum
'';
}

View file

@ -0,0 +1,3 @@
{
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
}

View file

@ -0,0 +1,8 @@
{ config, lib, ... }:
{
garage = lib.mkIf config.simulacrum {
keys.testkey = {};
buckets.testbucket.allow.testKey = [ "read" "write" ];
};
}

View file

@ -0,0 +1,105 @@
{ cluster, lib, ... }:
let
inherit (cluster.config.services.storage) nodes;
firstGarageNode = lib.elemAt nodes.garage 0;
in
{
nodes = lib.genAttrs nodes.garage (node: {
services.garage = {
layout.initial = lib.genAttrs nodes.garage (_: {
capacity = lib.mkOverride 51 1000;
});
};
specialisation.modifiedLayout = {
inheritParentConfig = true;
configuration = {
services.garage = {
layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
};
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
(i.runGarage ''
garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
garage layout apply --version 2
'')
]);
};
};
});
testScript = ''
import json
nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
garage1 = nodes[0]
start_all()
with subtest("should bootstrap new cluster"):
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
with subtest("should apply new layout with ascension"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
for node in nodes:
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
addr = consulConfig["addresses"]["http"]
port = consulConfig["ports"]["http"]
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
with subtest("should apply new layout from scratch"):
for node in nodes:
node.systemctl("stop garage.service")
node.succeed("rm -rf /var/lib/garage-metadata")
garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
for node in nodes:
node.systemctl("start garage.service")
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should create specified buckets and keys"):
for node in nodes:
node.wait_for_unit("incandescence-garage.target")
garage1.succeed("garage key list | grep testkey")
garage1.succeed("garage bucket list | grep testbucket")
with subtest("should delete unspecified keys"):
garage1.succeed("garage bucket create unwantedbucket")
garage1.succeed("garage key new --name unwantedkey")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage key list | grep unwantedkey")
garage1.succeed("garage bucket list | grep unwantedbucket")
with subtest("should delete unspecified buckets after grace period"):
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
garage1.succeed("systemctl restart garage.service")
garage1.wait_for_unit("incandescence-garage.target")
garage1.fail("garage bucket list | grep unwantedbucket")
'';
}

View file

@ -8,6 +8,7 @@
services.ways = {
nodes.host = config.services.websites.nodes.host;
nixos.host = ./host.nix;
simulacrum.deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
};
dns.records = lib.mapAttrs'

View file

@ -10,6 +10,19 @@ let
};
getExtAddr = host: host.interfaces.primary.addrPublic;
snakeoilPublicKeys = {
checkmate = "TESTtbFybW5YREwtd18a1A4StS4YAIUS5/M1Lv0jHjA=";
grail = "TEsTh7bthkaDh9A1CpqDi/F121ao5lRZqIJznLH8mB4=";
thunderskin = "tEST6afFmVN18o+EiWNFx+ax3MJwdQIeNfJSGEpffXw=";
VEGAS = "tEsT6s7VtM5C20eJBaq6UlQydAha8ATlmrTRe9T5jnM=";
prophet = "TEstYyb5IoqSL53HbSQwMhTaR16sxcWcMmXIBPd+1gE=";
};
grease = hourName: realPublicKey: if config.simulacrum then
snakeoilPublicKeys.${hourName}
else
realPublicKey;
in
{
vars = {
@ -22,7 +35,7 @@ in
extra = {
meshIp = "10.1.1.32";
inherit meshNet;
pubKey = "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
pubKey = grease "checkmate" "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
extraRoutes = [];
};
};
@ -31,7 +44,7 @@ in
extra = {
meshIp = "10.1.1.6";
inherit meshNet;
pubKey = "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
pubKey = grease "grail" "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
extraRoutes = [];
};
};
@ -40,7 +53,7 @@ in
extra = {
meshIp = "10.1.1.4";
inherit meshNet;
pubKey = "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
pubKey = grease "thunderskin" "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
extraRoutes = [];
};
};
@ -49,7 +62,7 @@ in
extra = {
meshIp = "10.1.1.5";
inherit meshNet;
pubKey = "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
pubKey = grease "VEGAS" "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
extraRoutes = [ "${hours.VEGAS.interfaces.vstub.addr}/32" "10.10.0.0/16" ];
};
};
@ -58,7 +71,7 @@ in
extra = {
meshIp = "10.1.1.9";
inherit meshNet;
pubKey = "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
pubKey = grease "prophet" "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
extraRoutes = [];
};
};
@ -69,8 +82,12 @@ in
storm = [ "VEGAS" ];
};
nixos = {
mesh = ./mesh.nix;
storm = ./storm.nix;
mesh = [
./mesh.nix
] ++ lib.optionals config.simulacrum [
./simulacrum/snakeoil-keys.nix
];
storm = [ ./storm.nix ];
};
secrets.meshPrivateKey = {
nodes = config.services.wireguard.nodes.mesh;

View file

@ -0,0 +1 @@
MNvWpMluuzQvPyGTp7jtyPSyz6n9lIly/WX1gW2NAHg=

View file

@ -0,0 +1 @@
YHzP8rBP6qiXs6ZdnvHop9KnCYRADIEejwZzAzvj8m4=

View file

@ -0,0 +1 @@
uD7X5E6N9d0sN+xPr/bWnehSa3bAok741GO7Z4I+Z3I=

View file

@ -0,0 +1 @@
QHyIJ3HoKGGFN28qOrQP4UyoQMP5bM7Idn2MzayKzEM=

View file

@ -0,0 +1 @@
YLl+hkWaCWx/5PpWs3cQ+bKqYdJef/qZ+FMTsM9ammM=

View file

@ -0,0 +1,6 @@
{ lib, config, ... }: {
config.environment.etc = {
"dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
"dummy-secrets/wireguard-key-storm".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
};
}

View file

@ -0,0 +1,16 @@
{ config, extendModules, lib, ... }:
{
perSystem = { pkgs, system, ... }: {
checks = lib.mkIf (system == "x86_64-linux") (lib.mapAttrs' (name: svc: let
runSimulacrum = pkgs.callPackage ./. {
inherit config extendModules;
};
in {
name = "simulacrum-${name}";
value = runSimulacrum {
service = name;
};
}) (lib.filterAttrs (_: svc: svc.simulacrum.enable) config.cluster.config.services));
};
}

View file

@ -0,0 +1,134 @@
{ testers, config, extendModules, lib, system }:
{ service }:
let
serviceConfig = config.cluster.config.services.${service};
serviceList = getDepsRecursive [] service;
allAugments = map (svc: config.cluster.config.services.${svc}.simulacrum.augments) serviceList;
getDepsRecursive = acc: service: let
deps = lib.subtractLists acc config.cluster.config.services.${service}.simulacrum.deps;
acc' = acc ++ [ service ];
recurse = getDepsRecursive acc';
in lib.unique (lib.flatten ([ service ] ++ map recurse deps));
lift = config;
snakeoil = {
ssh = {
public = lib.fileContents ../../packages/checks/snakeoil/ssh/snakeoil-key.pub;
private = ../../packages/checks/snakeoil/ssh/snakeoil-key;
};
};
nodes = lib.attrNames config.gods.fromLight;
nodes' = lib.attrNames (config.gods.fromLight // { nowhere = null; });
digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes' (lib.range 1 255));
depot' = extendModules {
modules = [
({ config, ... }: {
gods.fromLight = lib.mapAttrs (name: cfg: {
interfaces.primary = {
link = lib.mkForce "vprimary";
};
ssh.id.publicKey = lib.mkForce snakeoil.ssh.public;
}) lift.gods.fromLight;
cluster = lib.mkForce (lift.cluster.extendModules {
specialArgs.depot = config;
modules = [
{
simulacrum = true;
testConfig = {
subject = service;
activeServices = serviceList;
};
}
];
});
})
];
};
specialArgs = depot'.config.lib.summon system lib.id;
in
testers.runNixOSTest {
name = "simulacrum-${service}";
imports = [
serviceConfig.simulacrum.settings
./nowhere
{
nodes.nowhere.imports = [
config.flake.nixosModules.port-magic
];
}
] ++ allAugments;
_module.args = {
inherit (depot'.config) cluster;
};
node = { inherit specialArgs; };
nodes = lib.genAttrs nodes (node: let
hour = depot'.config.hours.${node};
in {
imports = [
specialArgs.depot.hours.${node}.nixos
../../packages/checks/modules/nixos/age-dummy-secrets
../../packages/checks/modules/nixos/external-storage.nix
] ++ depot'.config.cluster.config.out.injectNixosConfigForServices serviceList node;
boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
networking = {
interfaces = {
${hour.interfaces.primary.link} = {
useDHCP = lib.mkForce false;
virtual = true;
ipv4.addresses = lib.mkForce ([
{
address = hour.interfaces.primary.addr;
prefixLength = 32;
}
] ++ lib.optional hour.interfaces.primary.isNat {
address = hour.interfaces.primary.addrPublic;
prefixLength = 32;
});
};
eth1.ipv4.routes = lib.pipe nodes [
(lib.filter (n: n != node))
(map (n: let
hour = depot'.config.hours.${n};
in {
address = hour.interfaces.primary.addrPublic;
prefixLength = 32;
via = "192.168.1.${toString digits.${n}}";
}))
];
};
firewall.extraCommands = lib.mkAfter (lib.optionalString (hour.interfaces.primary.isNat) ''
# self-nat
iptables -t nat -A PREROUTING -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
iptables -t nat -A OUTPUT -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
iptables -t nat -A POSTROUTING -s ${hour.interfaces.primary.addr} -j SNAT --to-source ${hour.interfaces.primary.addrPublic}
'');
};
systemd.services = {
hyprspace.enable = false;
};
environment.etc = {
"ssh/ssh_host_ed25519_key" = {
source = snakeoil.ssh.private;
mode = "0400";
};
};
virtualisation = {
cores = 2;
memorySize = 4096;
};
});
}

View file

@ -0,0 +1,101 @@
{ cluster, config, lib, pkgs, ... }:
let
lift = config;
cfsslConfigIntermediateCA = pkgs.writeText "simulacrum-cfssl-config.json" (builtins.toJSON {
signing = {
default.expiry = "8760h";
profiles.intermediate = {
expiry = "8760h";
usages = [
"cert sign"
"crl sign"
];
ca_constraint = {
is_ca = true;
max_path_len = 1;
};
};
};
});
caCsr = pkgs.writeText "simulacrum-ca-csr.json" (builtins.toJSON {
CN = "Simulacrum Root CA";
});
ca = pkgs.runCommand "simulacrum-snakeoil-ca" {
nativeBuildInputs = [
pkgs.cfssl
];
} ''
mkdir $out
cfssl gencert --initca ${caCsr} | cfssljson --bare $out/ca
'';
genCert = extraFlags: csrData: let
csr = pkgs.writeText "simulacrum-csr.json" (builtins.toJSON csrData);
in pkgs.runCommand "simulacrum-snakeoil-cert" {
nativeBuildInputs = [
pkgs.cfssl
];
} ''
mkdir $out
cfssl gencert ${lib.escapeShellArgs ([
"--ca=file:${ca}/ca.pem"
"--ca-key=file:${ca}/ca-key.pem"
] ++ extraFlags ++ [
csr
])} | cfssljson --bare $out/cert
'';
genHostCert = hostname: genCert [ "--hostname=${hostname}" ] { CN = hostname; };
getNodeAddr = node: (builtins.head config.nodes.${node}.networking.interfaces.eth1.ipv4.addresses).address;
in
{
imports = [
./options.nix
];
defaults = {
networking.hosts."${getNodeAddr "nowhere"}" = lib.attrNames config.nowhere.names;
security.pki.certificateFiles = [
"${ca}/ca.pem"
];
};
nowhere.certs = {
inherit ca;
intermediate = genCert [ "--config=${cfsslConfigIntermediateCA}" "--profile=intermediate" ] {
CN = "Simulacrum Intermediate CA";
};
};
nodes.nowhere = { config, depot, ... }: {
networking = {
firewall.allowedTCPPorts = [ 443 ];
interfaces.eth1.ipv4.routes = lib.mapAttrsToList (name: hour: {
address = hour.interfaces.primary.addrPublic;
prefixLength = 32;
via = getNodeAddr name;
}) depot.gods.fromLight;
nameservers = map (name: depot.hours.${name}.interfaces.primary.addrPublic) cluster.config.services.dns.nodes.authoritative;
};
services.nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts = lib.mapAttrs (name: link: let
cert = genHostCert name;
in {
forceSSL = true;
sslCertificate = "${cert}/cert.pem";
sslCertificateKey = "${cert}/cert-key.pem";
locations."/" = {
proxyPass = config.links.${link}.url;
extraConfig = "proxy_ssl_verify off;";
};
}) lift.nowhere.names;
};
};
}

View file

@ -0,0 +1,16 @@
{ lib, ... }:
{
options.nowhere = {
names = lib.mkOption {
description = "Hostnames that point Nowhere.";
type = with lib.types; attrsOf str;
default = {};
};
certs = lib.mkOption {
description = "Snakeoil certificate packages.";
type = with lib.types; attrsOf package;
default = {};
};
};
}

View file

@ -8,6 +8,8 @@ let
cfgAge = config.age;
create = lib.flip lib.mapAttrs';
createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f;
in
{
@ -20,12 +22,17 @@ in
fileSystems = lib.mkOption {
description = "S3QL-based filesystems on top of CIFS mountpoints.";
default = {};
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: {
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let
authFile = if config.locksmithSecret != null then
"/run/locksmith/${config.locksmithSecret}"
else
cfgAge.secrets."storageAuth-${name}".path;
in {
imports = [ ./filesystem-type.nix ];
backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}";
commonArgs = [
"--cachedir" config.cacheDir
"--authfile" cfgAge.secrets."storageAuth-${name}".path
"--authfile" authFile
] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]);
}));
};
@ -57,9 +64,14 @@ in
age.secrets = lib.mkMerge [
(create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; }))
(create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
(createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
];
services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: {
name = fs.unitName;
value = [ fs.locksmithSecret ];
});
fileSystems = create cfg.underlays (name: ul: {
name = ul.mountpoint;
value = {
@ -97,7 +109,13 @@ in
value = let
isUnderlay = fs.underlay != null;
fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);
backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
fsType = if isUnderlay then "local" else lib.head backendParts;
s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
in {
@ -120,8 +138,12 @@ in
ExecStartPre = map lib.escapeShellArgs [
[
(let
authFile = if fs.locksmithSecret != null then
"/run/locksmith/${fs.locksmithSecret}"
else
cfgAge.secrets."storageAuth-${name}".path;
mkfsEncrypted = ''
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
| cut -d' ' -f2- \
| ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
'';
@ -132,6 +154,11 @@ in
detectFs = {
local = "test -e ${localBackendPath}/s3ql_metadata";
s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
'';
}.${fsType} or null;
in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
if ! ${detectFs}; then

View file

@ -22,6 +22,10 @@ with lib;
authFile = mkOption {
type = types.path;
};
locksmithSecret = mkOption {
type = with types; nullOr str;
default = null;
};
cacheDir = mkOption {
type = types.path;
default = "/var/cache/remote-storage/${name}";

View file

@ -1,21 +1,32 @@
{ lib, ... }:
{
perSystem = { config, ... }: {
catalog.depot = {
checks = lib.mapAttrs (name: check: {
description = "NixOS Test: ${name}";
actions = {
build = {
description = "Build this check.";
command = "nix build -L --no-link '${builtins.unsafeDiscardStringContext check.drvPath}^*'";
};
runInteractive = {
description = "Run interactive driver.";
command = lib.getExe check.driverInteractive;
};
perSystem = { config, pkgs, ... }: {
catalog = lib.mkMerge (lib.mapAttrsToList (name': check: let
simulacrum = lib.hasPrefix "simulacrum-" name';
name = lib.removePrefix "simulacrum-" name';
baseAttrPath = if simulacrum then
[ "cluster" "simulacrum" ]
else
[ "depot" "checks" ];
in lib.setAttrByPath (baseAttrPath ++ [ name ]) {
description = if simulacrum then
"Simulacrum Test: ${name}"
else
"NixOS Test: ${name}";
actions = {
build = {
description = "Build this check.";
command = "nix build -L --no-link '${builtins.unsafeDiscardStringContext check.drvPath}^*'";
};
}) config.checks;
};
runInteractive = {
description = "Run interactive driver.";
command = if simulacrum then
"${pkgs.bubblewrap}/bin/bwrap --unshare-all --bind / / --dev-bind /dev /dev ${lib.getExe check.driverInteractive}"
else
lib.getExe check.driverInteractive;
};
};
}) config.checks);
};
}

View file

@ -7,6 +7,7 @@ let
in
{
debug = lib.warn "debug mode is enabled" true;
perSystem = { filters, pkgs, self', system, ... }: {
checks = lib.mkIf (system == "x86_64-linux") {
ascensions = pkgs.callPackage ./ascensions.nix {
@ -15,12 +16,6 @@ in
inherit (config) cluster;
};
garage = pkgs.callPackage ./garage.nix {
inherit (self'.packages) garage consul;
inherit (self) nixosModules;
inherit (config) cluster;
};
ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
inherit (self) nixosModules;
previous = timeMachine.preUnstable;

View file

@ -1,155 +0,0 @@
{ testers, nixosModules, cluster, garage, consul }:
testers.runNixOSTest {
name = "garage";
imports = [
./modules/consul.nix
];
extraBaseModules.services.consul.package = consul;
nodes = let
common = { config, lib, ... }: let
inherit (config.networking) hostName primaryIPAddress;
in {
imports = lib.flatten [
./modules/nixos/age-dummy-secrets
./modules/nixos/age-dummy-secrets/options.nix
nixosModules.ascensions
nixosModules.systemd-extras
nixosModules.consul-distributed-services
nixosModules.port-magic
cluster.config.services.storage.nixos.garage
cluster.config.services.storage.nixos.garageInternal
cluster.config.services.consul.nixos.ready
];
options.services.locksmith.providers = lib.mkOption {
type = lib.types.raw;
};
config = {
links.consulAgent = {
protocol = "http";
hostname = "consul";
port = 8500;
};
_module.args = {
depot.packages = { inherit garage; };
cluster.config = {
hostLinks.${hostName} = {
garageRpc.tuple = "${primaryIPAddress}:3901";
garageS3.tuple = "${primaryIPAddress}:8080";
garageWeb.tuple = "${primaryIPAddress}:8081";
};
links.garageWeb.hostname = "web.garage.example.com";
vars.meshNet.cidr = "192.168.0.0/16";
};
};
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
networking.firewall.allowedTCPPorts = [ 3901 8080 ];
services.garage = {
layout.initial = lib.mkOverride 51 {
garage1 = { zone = "dc1"; capacity = 1000; };
garage2 = { zone = "dc1"; capacity = 1000; };
garage3 = { zone = "dc1"; capacity = 1000; };
};
};
system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
specialisation.modifiedLayout = {
inheritParentConfig = true;
configuration = {
services.garage = {
layout.initial = lib.mkForce {
garage1 = { zone = "dc1"; capacity = 2000; };
garage2 = { zone = "dc1"; capacity = 1000; };
garage3 = { zone = "dc1"; capacity = 1000; };
};
keys.testKey.allow.createBucket = true;
buckets = {
bucket1 = {
allow.testKey = [ "read" "write" ];
quotas = {
maxObjects = 300;
maxSize = 400 * 1024 * 1024;
};
};
bucket2 = {
allow.testKey = [ "read" ];
};
};
};
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
(i.runGarage ''
garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
garage layout apply --version 2
'')
]);
};
};
};
};
in {
garage1.imports = [ common ];
garage2.imports = [ common ];
garage3.imports = [ common ];
};
testScript = { nodes, ... }: /*python*/ ''
nodes = [garage1, garage2, garage3]
start_all()
with subtest("should bootstrap new cluster"):
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
with subtest("should apply new layout with ascension"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
for node in nodes:
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should apply new layout from scratch"):
for node in nodes:
node.systemctl("stop garage.service")
node.succeed("rm -rf /var/lib/garage-metadata")
for node in nodes:
node.systemctl("start garage.service")
for node in nodes:
node.wait_for_unit("garage.service")
for node in nodes:
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
for node in nodes:
node.wait_until_succeeds("garage layout show | grep -w 2000")
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
with subtest("should create specified buckets and keys"):
for node in nodes:
node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
garage1.succeed("garage key list | grep testKey")
garage1.succeed("garage bucket list | grep bucket1")
garage1.succeed("garage bucket list | grep bucket2")
with subtest("should delete unspecified buckets and keys"):
garage1.succeed("garage bucket create unwantedbucket")
garage1.succeed("garage key new --name unwantedkey")
garage1.succeed("systemctl restart garage-apply.service")
garage1.fail("garage key list | grep unwantedkey")
garage1.fail("garage bucket list | grep unwantedbucket")
'';
}

View file

@ -0,0 +1,12 @@
{ config, lib, ... }:
{
systemd.tmpfiles.settings."00-testing-external-storage-underlays" = lib.mapAttrs' (name: cfg: {
name = cfg.mountpoint;
value.d = {
user = toString cfg.uid;
group = toString cfg.gid;
mode = "0700";
};
}) config.services.external-storage.underlays;
}

View file

@ -0,0 +1,7 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgwAAAJAS78fWEu/H
1gAAAAtzc2gtZWQyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgw
AAAEAUtGOZZIZdzGP6g85JuXBjDtciNQ9bLHNxSN5Gbwvb2Q7HTdf4u1bRo3x6N03ggmAM
+tNmBXB7tZteGEG+pXCDAAAACW1heEBUSVRBTgECAwQ=
-----END OPENSSH PRIVATE KEY-----

View file

@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA7HTdf4u1bRo3x6N03ggmAM+tNmBXB7tZteGEG+pXCD