Compare commits
52 commits
master
...
pr-frangic
Author | SHA1 | Date | |
---|---|---|---|
00f233e8a5 | |||
e2fe73039c | |||
e4f09218d5 | |||
ebc9b88c8d | |||
ea3414c427 | |||
6eee030b7a | |||
75cebf4ed6 | |||
bc3cd82731 | |||
9cdf964c6c | |||
bb22fe0176 | |||
d1f2bc1227 | |||
a48ee00f3d | |||
9ed3655ccf | |||
945698a3ea | |||
f75c7b8522 | |||
b258bab23e | |||
e2296eace7 | |||
304ae6e53c | |||
f322208f66 | |||
7c4615ecfb | |||
ec38e10fa9 | |||
5d9ff62afe | |||
6d78b69601 | |||
7129d44078 | |||
76d205d114 | |||
c8c9a6fbce | |||
a1cad2efcd | |||
c7f4e59908 | |||
baed1ce871 | |||
014c1f9cd2 | |||
34704c8f08 | |||
ccc2a47880 | |||
05cd729e90 | |||
8d0a2f00cc | |||
ff26e1ebc1 | |||
b848084dd8 | |||
fe8ddd4094 | |||
030b680b33 | |||
b453b0bb21 | |||
b6e0390555 | |||
bbe3373c2e | |||
0ed4870b65 | |||
8ec13f5c87 | |||
5d52f72940 | |||
1af67b80ed | |||
a810717843 | |||
bd39fc5d07 | |||
25c001c182 | |||
d944dee3bc | |||
aac5163a8b | |||
01c74f62cf | |||
0110a4a0c3 |
67 changed files with 1333 additions and 358 deletions
|
@ -46,6 +46,7 @@ in
|
|||
};
|
||||
}) // (if secretConfig.shared then let
|
||||
secretFile = "${svcName}-${secretName}.age";
|
||||
snakeoilFile = "${svcName}-${secretName}-snakeoil.txt";
|
||||
in {
|
||||
editSecret = {
|
||||
description = "Edit this secret";
|
||||
|
@ -54,15 +55,31 @@ in
|
|||
agenix -e '${secretFile}'
|
||||
'';
|
||||
};
|
||||
} else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
|
||||
secretFile = "${svcName}-${secretName}-${node}.age";
|
||||
in {
|
||||
description = "Edit this secret for '${node}'";
|
||||
command = ''
|
||||
${setupCommands secretFile [ node ]}
|
||||
agenix -e '${secretFile}'
|
||||
'';
|
||||
})));
|
||||
editSnakeoil = {
|
||||
description = "Edit this secret's snakeoil";
|
||||
command = ''
|
||||
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
|
||||
'';
|
||||
};
|
||||
} else lib.mkMerge [
|
||||
(lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
|
||||
secretFile = "${svcName}-${secretName}-${node}.age";
|
||||
in {
|
||||
description = "Edit this secret for '${node}'";
|
||||
command = ''
|
||||
${setupCommands secretFile [ node ]}
|
||||
agenix -e '${secretFile}'
|
||||
'';
|
||||
})))
|
||||
(lib.mapAttrs' (name: lib.nameValuePair "editSnakeoilInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
|
||||
snakeoilFile = "${svcName}-${secretName}-${node}-snakeoil.txt";
|
||||
in {
|
||||
description = "Edit this secret's snakeoil for '${node}'";
|
||||
command = ''
|
||||
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
|
||||
'';
|
||||
})))
|
||||
]);
|
||||
};
|
||||
}) svcConfig.secrets))
|
||||
lib.concatLists
|
||||
|
|
|
@ -16,6 +16,8 @@ lib.evalModules {
|
|||
./lib/port-magic-multi.nix
|
||||
./lib/mesh.nix
|
||||
./lib/secrets.nix
|
||||
./lib/testing.nix
|
||||
./lib/lib.nix
|
||||
|
||||
./import-services.nix
|
||||
];
|
||||
|
|
|
@ -2,9 +2,9 @@
|
|||
with lib;
|
||||
|
||||
{
|
||||
options.out.injectNixosConfig = mkOption {
|
||||
description = "NixOS configuration to inject into the given host.";
|
||||
type = with types; functionTo raw;
|
||||
options.out = mkOption {
|
||||
description = "Output functions.";
|
||||
type = with types; lazyAttrsOf (functionTo raw);
|
||||
default = const [];
|
||||
};
|
||||
}
|
||||
|
|
12
cluster/lib/lib.nix
Normal file
12
cluster/lib/lib.nix
Normal file
|
@ -0,0 +1,12 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
options.lib = {
|
||||
forService = lib.mkOption {
|
||||
description = "Enable these definitions for a particular service only.";
|
||||
type = lib.types.functionTo lib.types.raw;
|
||||
readOnly = true;
|
||||
default = service: lib.mkIf (!config.simulacrum || lib.any (s: s == service) config.testConfig.activeServices);
|
||||
};
|
||||
};
|
||||
}
|
|
@ -52,6 +52,24 @@ in
|
|||
}));
|
||||
default = {};
|
||||
};
|
||||
simulacrum = {
|
||||
enable = mkEnableOption "testing this service in the Simulacrum";
|
||||
deps = mkOption {
|
||||
description = "Other services to include.";
|
||||
type = with types; listOf str;
|
||||
default = [];
|
||||
};
|
||||
settings = mkOption {
|
||||
description = "NixOS test configuration.";
|
||||
type = types.deferredModule;
|
||||
default = {};
|
||||
};
|
||||
augments = mkOption {
|
||||
description = "Cluster augments (will be propagated).";
|
||||
type = types.deferredModule;
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
};
|
||||
config.otherNodes = builtins.mapAttrs (const filterGroup) config.nodes;
|
||||
}
|
||||
|
|
|
@ -39,7 +39,11 @@ in
|
|||
default = {};
|
||||
};
|
||||
|
||||
config.out.injectNixosConfig = hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) config.services)) ++ [
|
||||
introspectionModule
|
||||
];
|
||||
config.out = {
|
||||
injectNixosConfigForServices = services: hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) (lib.getAttrs services config.services))) ++ [
|
||||
introspectionModule
|
||||
];
|
||||
|
||||
injectNixosConfig = config.out.injectNixosConfigForServices (lib.attrNames config.services);
|
||||
};
|
||||
}
|
||||
|
|
15
cluster/lib/testing.nix
Normal file
15
cluster/lib/testing.nix
Normal file
|
@ -0,0 +1,15 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
options = {
|
||||
simulacrum = lib.mkOption {
|
||||
description = "Whether we are in the Simulacrum.";
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
};
|
||||
testConfig = lib.mkOption {
|
||||
type = lib.types.attrs;
|
||||
readOnly = true;
|
||||
};
|
||||
};
|
||||
}
|
|
@ -3,6 +3,7 @@
|
|||
{
|
||||
imports = [
|
||||
./catalog
|
||||
./simulacrum/checks.nix
|
||||
];
|
||||
|
||||
options.cluster = lib.mkOption {
|
||||
|
|
|
@ -1,13 +0,0 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
|
||||
JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
|
||||
-> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
|
||||
2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
|
||||
-> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
|
||||
bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
|
||||
-> Ovax-grease ^1$]}H G4 FpDF XKHkj{
|
||||
IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
|
||||
--- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
|
||||
ÜA³×oÈð:±‹`ÜVd±å(Kät:fk¼’}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69i›l`ÛÆJKwAè8y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
|
||||
üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ý’wÊ
|
||||
ÆÜßÌZî!^þRˆéÿv¾…ïk‹Êp»ÛPÌ)ý̆ÍpÓV5²F΄ÆÚÙÚÞhBÇ»ßb#Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè‹}€ß ƒ¬E}~ZHûjmyq{òxŠ–Éôß<C3B4>"”éÀ´C#šójÿÐ.ò§yÔ£¸v¦
<0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që
|
60
cluster/services/acme-client/augment.nix
Normal file
60
cluster/services/acme-client/augment.nix
Normal file
|
@ -0,0 +1,60 @@
|
|||
{ config, pkgs, ... }:
|
||||
|
||||
let
|
||||
lift = config;
|
||||
in
|
||||
|
||||
{
|
||||
nowhere.names = {
|
||||
"acme-v02.api.letsencrypt.org" = "stepCa";
|
||||
"api.buypass.com" = "stepCa";
|
||||
};
|
||||
|
||||
nodes.nowhere = { config, ... }: {
|
||||
links.stepCa.protocol = "https";
|
||||
|
||||
environment.etc.step-ca-password.text = "";
|
||||
|
||||
services = {
|
||||
step-ca = {
|
||||
enable = true;
|
||||
address = config.links.stepCa.ipv4;
|
||||
inherit (config.links.stepCa) port;
|
||||
intermediatePasswordFile = "/etc/step-ca-password";
|
||||
settings = {
|
||||
root = "${lift.nowhere.certs.ca}/ca.pem";
|
||||
crt = "${lift.nowhere.certs.intermediate}/cert.pem";
|
||||
key = "${lift.nowhere.certs.intermediate}/cert-key.pem";
|
||||
address = config.links.stepCa.tuple;
|
||||
db = {
|
||||
type = "badgerv2";
|
||||
dataSource = "/var/lib/step-ca/db";
|
||||
};
|
||||
authority.provisioners = [
|
||||
{
|
||||
type = "ACME";
|
||||
name = "snakeoil";
|
||||
challenges = [
|
||||
"dns-01"
|
||||
"http-01"
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
nginx.virtualHosts = {
|
||||
"acme-v02.api.letsencrypt.org".locations."/".extraConfig = ''
|
||||
rewrite /directory /acme/snakeoil/directory break;
|
||||
'';
|
||||
"api.buypass.com".locations."/".extraConfig = ''
|
||||
rewrite /acme/directory /acme/snakeoil/directory break;
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
defaults.environment.etc."dummy-secrets/acmeDnsApiKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
|
||||
defaults.environment.etc."dummy-secrets/acmeDnsDirectKey".text = "ACME_DNS_DIRECT_STATIC_KEY=simulacrum";
|
||||
defaults.environment.etc."dummy-secrets/acmeDnsDbCredentials".text = "PGPASSWORD=simulacrum";
|
||||
}
|
|
@ -2,5 +2,6 @@
|
|||
services.acme-client = {
|
||||
nodes.client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
|
||||
nixos.client = ./client.nix;
|
||||
simulacrum.augments = ./augment.nix;
|
||||
};
|
||||
}
|
||||
|
|
|
@ -33,7 +33,7 @@
|
|||
};
|
||||
};
|
||||
|
||||
garage = {
|
||||
garage = config.lib.forService "attic" {
|
||||
keys.attic.locksmith = {
|
||||
nodes = config.services.attic.nodes.server;
|
||||
owner = "atticd";
|
||||
|
@ -48,7 +48,7 @@
|
|||
serverAddrs = map
|
||||
(node: depot.hours.${node}.interfaces.primary.addrPublic)
|
||||
config.services.attic.nodes.server;
|
||||
in {
|
||||
in config.lib.forService "attic" {
|
||||
cache.target = serverAddrs;
|
||||
};
|
||||
|
||||
|
|
|
@ -6,5 +6,6 @@
|
|||
nixos.listener = [
|
||||
./listener.nix
|
||||
];
|
||||
simulacrum.deps = [ "consul" ];
|
||||
};
|
||||
}
|
||||
|
|
|
@ -10,6 +10,13 @@ let
|
|||
in
|
||||
|
||||
{
|
||||
system.extraIncantations = {
|
||||
runConsul = i: script: i.execShellWith [ config.services.consul.package ] ''
|
||||
export CONSUL_HTTP_ADDR='${config.links.consulAgent.tuple}'
|
||||
${script}
|
||||
'';
|
||||
};
|
||||
|
||||
links.consulAgent.protocol = "http";
|
||||
|
||||
services.consul = {
|
||||
|
|
|
@ -22,6 +22,11 @@ in
|
|||
];
|
||||
ready = ./ready.nix;
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "wireguard" ];
|
||||
settings = ./test.nix;
|
||||
};
|
||||
};
|
||||
|
||||
dns.records."consul-remote.internal".consulService = "consul-remote";
|
||||
|
|
19
cluster/services/consul/test.nix
Normal file
19
cluster/services/consul/test.nix
Normal file
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
testScript = ''
|
||||
import json
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should form cluster"):
|
||||
for machine in machines:
|
||||
machine.succeed("systemctl start consul-ready.service")
|
||||
for machine in machines:
|
||||
consulConfig = json.loads(machine.succeed("cat /etc/consul.json"))
|
||||
addr = consulConfig["addresses"]["http"]
|
||||
port = consulConfig["ports"]["http"]
|
||||
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
|
||||
memberList = machine.succeed(f"{setEnv} consul members --status=alive")
|
||||
for machine2 in machines:
|
||||
assert machine2.name in memberList
|
||||
'';
|
||||
}
|
|
@ -43,9 +43,6 @@ in {
|
|||
links.localAuthoritativeDNS = {};
|
||||
|
||||
age.secrets = {
|
||||
acmeDnsDbCredentials = {
|
||||
file = ./acme-dns-db-credentials.age;
|
||||
};
|
||||
acmeDnsDirectKey = {
|
||||
file = ./acme-dns-direct-key.age;
|
||||
};
|
||||
|
@ -78,8 +75,12 @@ in {
|
|||
};
|
||||
};
|
||||
|
||||
services.locksmith.waitForSecrets.acme-dns = [
|
||||
"patroni-acmedns"
|
||||
];
|
||||
|
||||
systemd.services.acme-dns.serviceConfig.EnvironmentFile = with config.age.secrets; [
|
||||
acmeDnsDbCredentials.path
|
||||
"/run/locksmith/patroni-acmedns"
|
||||
acmeDnsDirectKey.path
|
||||
];
|
||||
|
||||
|
|
|
@ -58,6 +58,16 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
patroni = {
|
||||
databases.acmedns = {};
|
||||
users.acmedns = {
|
||||
locksmith = {
|
||||
nodes = config.services.dns.nodes.authoritative;
|
||||
format = "envFile";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
dns.records = {
|
||||
securedns.consulService = "securedns";
|
||||
"acme-dns-challenge.internal".consulService = "acme-dns";
|
||||
|
|
|
@ -13,25 +13,36 @@
|
|||
nodes = server;
|
||||
owner = "forgejo";
|
||||
};
|
||||
dbCredentials.nodes = server;
|
||||
};
|
||||
};
|
||||
|
||||
ways.forge.target = let
|
||||
ways.forge = let
|
||||
host = builtins.head config.services.forge.nodes.server;
|
||||
in config.hostLinks.${host}.forge.url;
|
||||
in config.lib.forService "forge" {
|
||||
target = config.hostLinks.${host}.forge.url;
|
||||
};
|
||||
|
||||
garage = {
|
||||
patroni = config.lib.forService "forge" {
|
||||
databases.forge = {};
|
||||
users.forge.locksmith = {
|
||||
nodes = config.services.forge.nodes.server;
|
||||
format = "raw";
|
||||
};
|
||||
};
|
||||
|
||||
garage = config.lib.forService "forge" {
|
||||
keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
|
||||
buckets.forgejo.allow.forgejo = [ "read" "write" ];
|
||||
};
|
||||
|
||||
monitoring.blackbox.targets.forge = {
|
||||
monitoring.blackbox.targets.forge = config.lib.forService "forge" {
|
||||
address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
|
||||
module = "https2xx";
|
||||
};
|
||||
|
||||
dns.records."ssh.forge".target = map
|
||||
(node: depot.hours.${node}.interfaces.primary.addrPublic)
|
||||
config.services.forge.nodes.server;
|
||||
dns.records = config.lib.forService "forge" {
|
||||
"ssh.forge".target = map
|
||||
(node: depot.hours.${node}.interfaces.primary.addrPublic)
|
||||
config.services.forge.nodes.server;
|
||||
};
|
||||
}
|
||||
|
|
|
@ -26,6 +26,7 @@ in
|
|||
services.locksmith.waitForSecrets.forgejo = [
|
||||
"garage-forgejo-id"
|
||||
"garage-forgejo-secret"
|
||||
"patroni-forge"
|
||||
];
|
||||
|
||||
services.forgejo = {
|
||||
|
@ -39,7 +40,7 @@ in
|
|||
inherit (patroni) port;
|
||||
name = "forge";
|
||||
user = "forge";
|
||||
passwordFile = secrets.dbCredentials.path;
|
||||
passwordFile = "/run/locksmith/patroni-forge";
|
||||
};
|
||||
settings = {
|
||||
DEFAULT = {
|
||||
|
|
27
cluster/services/frangiclave/default.nix
Normal file
27
cluster/services/frangiclave/default.nix
Normal file
|
@ -0,0 +1,27 @@
|
|||
{ config, ... }:
|
||||
|
||||
{
|
||||
services.frangiclave = {
|
||||
nodes = {
|
||||
server = [ "VEGAS" "grail" "prophet" ];
|
||||
cluster = config.services.frangiclave.nodes.server;
|
||||
agent = []; # all nodes, for vault-agent, secret templates, etc.
|
||||
};
|
||||
meshLinks = {
|
||||
server.link.protocol = "http";
|
||||
cluster.link.protocol = "http";
|
||||
};
|
||||
nixos = {
|
||||
server = [
|
||||
./server.nix
|
||||
];
|
||||
cluster = [];
|
||||
agent = [];
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "wireguard" "consul" ];
|
||||
settings = ./test.nix;
|
||||
};
|
||||
};
|
||||
}
|
34
cluster/services/frangiclave/server.nix
Normal file
34
cluster/services/frangiclave/server.nix
Normal file
|
@ -0,0 +1,34 @@
|
|||
{ cluster, config, depot, lib, ... }:
|
||||
|
||||
let
|
||||
apiLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server;
|
||||
clusterLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-cluster;
|
||||
in
|
||||
|
||||
{
|
||||
services.vault = {
|
||||
enable = true;
|
||||
package = depot.packages.openbao;
|
||||
address = apiLink.tuple;
|
||||
extraConfig = /*hcl*/ ''
|
||||
api_addr = "${apiLink.url}"
|
||||
cluster_addr = "${clusterLink.url}"
|
||||
'';
|
||||
|
||||
storageBackend = "raft";
|
||||
storageConfig = /*hcl*/ ''
|
||||
node_id = "x${builtins.hashString "sha256" "frangiclave-node-${config.networking.hostName}"}"
|
||||
${
|
||||
lib.pipe (cluster.config.services.frangiclave.otherNodes.server config.networking.hostName) [
|
||||
(map (node: cluster.config.hostLinks.${node}.frangiclave-server))
|
||||
(map (link: /*hcl*/ ''
|
||||
retry_join {
|
||||
leader_api_addr = "${link.url}"
|
||||
}
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]
|
||||
}
|
||||
'';
|
||||
};
|
||||
}
|
12
cluster/services/frangiclave/test.nix
Normal file
12
cluster/services/frangiclave/test.nix
Normal file
|
@ -0,0 +1,12 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
interactive.defaults = { cluster, config, ... }: {
|
||||
config = lib.mkIf config.services.vault.enable {
|
||||
environment.variables.VAULT_ADDR = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server.url;
|
||||
environment.systemPackages = [ config.services.vault.package ];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = "assert False";
|
||||
}
|
|
@ -62,7 +62,7 @@
|
|||
lib.unique
|
||||
(map (x: "hci-agent-${x}"))
|
||||
];
|
||||
in {
|
||||
in config.lib.forService "hercules-ci-multi-agent" {
|
||||
keys = lib.genAttrs hciAgentKeys (lib.const {});
|
||||
buckets.nix-store = {
|
||||
allow = lib.genAttrs hciAgentKeys (lib.const [ "read" "write" ]);
|
||||
|
|
20
cluster/services/incandescence/default.nix
Normal file
20
cluster/services/incandescence/default.nix
Normal file
|
@ -0,0 +1,20 @@
|
|||
{ config, ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
./options.nix
|
||||
];
|
||||
|
||||
services.incandescence = {
|
||||
nodes = {
|
||||
provider = config.services.consul.nodes.agent;
|
||||
};
|
||||
nixos = {
|
||||
provider = [
|
||||
./provider.nix
|
||||
./provider-options.nix
|
||||
];
|
||||
};
|
||||
simulacrum.deps = [ "consul" ];
|
||||
};
|
||||
}
|
22
cluster/services/incandescence/options.nix
Normal file
22
cluster/services/incandescence/options.nix
Normal file
|
@ -0,0 +1,22 @@
|
|||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) mkOption;
|
||||
inherit (lib.types) attrsOf listOf submodule str;
|
||||
in
|
||||
|
||||
{
|
||||
options.incandescence = {
|
||||
providers = mkOption {
|
||||
type = attrsOf (submodule ({ name, ... }: {
|
||||
options = {
|
||||
objects = mkOption {
|
||||
type = attrsOf (listOf str);
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
}));
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
}
|
72
cluster/services/incandescence/provider-options.nix
Normal file
72
cluster/services/incandescence/provider-options.nix
Normal file
|
@ -0,0 +1,72 @@
|
|||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) mkEnableOption mkOption;
|
||||
inherit (lib.types) attrsOf functionTo ints listOf nullOr package submodule str;
|
||||
in
|
||||
|
||||
{
|
||||
options.services.incandescence = {
|
||||
providers = mkOption {
|
||||
type = attrsOf (submodule ({ name, ... }: {
|
||||
options = {
|
||||
locksmith = mkEnableOption "Locksmith integration";
|
||||
|
||||
wantedBy = mkOption {
|
||||
type = listOf str;
|
||||
};
|
||||
|
||||
partOf = mkOption {
|
||||
type = listOf str;
|
||||
};
|
||||
|
||||
wants = mkOption {
|
||||
type = listOf str;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
after = mkOption {
|
||||
type = listOf str;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
packages = mkOption {
|
||||
type = listOf package;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
formulae = mkOption {
|
||||
type = attrsOf (submodule ({ ... }: {
|
||||
options = {
|
||||
deps = mkOption {
|
||||
type = listOf str;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
create = mkOption {
|
||||
type = functionTo str;
|
||||
};
|
||||
|
||||
change = mkOption {
|
||||
type = nullOr (functionTo str);
|
||||
default = null;
|
||||
};
|
||||
|
||||
destroy = mkOption {
|
||||
type = str;
|
||||
};
|
||||
|
||||
destroyAfterDays = mkOption {
|
||||
type = ints.unsigned;
|
||||
default = 0;
|
||||
};
|
||||
};
|
||||
}));
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
}));
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
}
|
138
cluster/services/incandescence/provider.nix
Normal file
138
cluster/services/incandescence/provider.nix
Normal file
|
@ -0,0 +1,138 @@
|
|||
{ cluster, config, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) concatStringsSep escapeShellArg flatten filter filterAttrs length mapAttrs mapAttrs' mapAttrsToList mkIf mkMerge pipe stringToCharacters;
|
||||
|
||||
cfg = config.services.incandescence;
|
||||
clusterCfg = cluster.config.incandescence;
|
||||
in
|
||||
|
||||
{
|
||||
systemd.services = pipe cfg.providers [
|
||||
(mapAttrsToList (provider: providerConfig: pipe providerConfig.formulae [
|
||||
(mapAttrsToList (formula: formulaConfig: let
|
||||
kvRoot = "services/incandescence/providers/${provider}/formulae/${formula}";
|
||||
time = "$(date +%s)";
|
||||
in {
|
||||
"ignite-${provider}-${formula}-create" = {
|
||||
description = "Ignite Creation: ${provider} - ${formula}";
|
||||
wantedBy = [ "incandescence-${provider}.target" ];
|
||||
before = [ "incandescence-${provider}.target" ];
|
||||
wants = providerConfig.wants ++ map (dep: "ignite-${provider}-${dep}-create.service") formulaConfig.deps;
|
||||
after = providerConfig.after ++ map (dep: "ignite-${provider}-${dep}-create.service") formulaConfig.deps;
|
||||
serviceConfig.Type = "oneshot";
|
||||
distributed.enable = true;
|
||||
path = [ config.services.consul.package ] ++ providerConfig.packages;
|
||||
script = pipe clusterCfg.providers.${provider}.objects.${formula} [
|
||||
(map (object: ''
|
||||
if ! consul kv get ${kvRoot}/${object}/alive >/dev/null; then
|
||||
echo "Create ${formula}: ${object}"
|
||||
if (
|
||||
${formulaConfig.create object}
|
||||
)
|
||||
then
|
||||
consul kv put ${kvRoot}/${object}/alive true
|
||||
consul kv delete ${kvRoot}/${object}/destroyOn
|
||||
else
|
||||
echo "Creation failed: ${object}"
|
||||
fi
|
||||
fi
|
||||
''))
|
||||
(concatStringsSep "\n")
|
||||
];
|
||||
};
|
||||
"ignite-${provider}-${formula}-change" = mkIf (formulaConfig.change != null) {
|
||||
description = "Ignite Change: ${provider} - ${formula}";
|
||||
wantedBy = [ "incandescence-${provider}.target" ];
|
||||
before = [ "incandescence-${provider}.target" ];
|
||||
wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-create.service" ] ++ map (dep: "ignite-${provider}-${dep}-change.service") formulaConfig.deps;
|
||||
after = providerConfig.after ++ [ "ignite-${provider}-${formula}-create.service" ] ++ map (dep: "ignite-${provider}-${dep}-change.service") formulaConfig.deps;
|
||||
serviceConfig.Type = "oneshot";
|
||||
distributed.enable = true;
|
||||
path = [ config.services.consul.package ] ++ providerConfig.packages;
|
||||
script = pipe clusterCfg.providers.${provider}.objects.${formula} [
|
||||
(map (object: ''
|
||||
echo "Change ${formula}: ${object}"
|
||||
(
|
||||
${formulaConfig.change object}
|
||||
) || echo "Change failed: ${object}"
|
||||
''))
|
||||
(concatStringsSep "\n")
|
||||
];
|
||||
};
|
||||
"ignite-${provider}-${formula}-destroy" = {
|
||||
description = "Ignite Destruction: ${provider} - ${formula}";
|
||||
wantedBy = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
|
||||
before = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
|
||||
wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-change.service" ];
|
||||
after = providerConfig.after ++ [ "ignite-${provider}-${formula}-change.service" ];
|
||||
serviceConfig.Type = "oneshot";
|
||||
distributed.enable = true;
|
||||
path = [ config.services.consul.package ] ++ providerConfig.packages;
|
||||
script = let
|
||||
fieldNum = pipe kvRoot [
|
||||
stringToCharacters
|
||||
(filter (x: x == "/"))
|
||||
length
|
||||
(builtins.add 2)
|
||||
toString
|
||||
];
|
||||
keyFilter = pipe clusterCfg.providers.${provider}.objects.${formula} [
|
||||
(map (x: escapeShellArg "^${x}$"))
|
||||
(concatStringsSep " \\\n -e ")
|
||||
];
|
||||
destroyAfterDays = toString formulaConfig.destroyAfterDays;
|
||||
in ''
|
||||
consul kv get --keys ${kvRoot}/ | cut -d/ -f${fieldNum} | grep -v -e ${keyFilter} | while read object; do
|
||||
if consul kv get ${kvRoot}/$object/alive >/dev/null; then
|
||||
destroyOn="$(consul kv get ${kvRoot}/$object/destroyOn || true)"
|
||||
if [[ -z "$destroyOn" && "${destroyAfterDays}" -ne 0 ]]; then
|
||||
echo "Schedule ${formula} for destruction in ${destroyAfterDays} days: $object"
|
||||
consul kv put ${kvRoot}/$object/destroyOn "$((${time} + 86400 * ${destroyAfterDays}))"
|
||||
elif [[ "${destroyAfterDays}" -eq 0 || "${time}" -ge "$destroyOn" ]]; then
|
||||
echo "Destroy ${formula}: $object"
|
||||
export OBJECT="$object"
|
||||
if (
|
||||
${formulaConfig.destroy}
|
||||
)
|
||||
then
|
||||
consul kv delete --recurse ${kvRoot}/$object
|
||||
else
|
||||
echo "Destruction failed: $object"
|
||||
fi
|
||||
else
|
||||
echo "Scheduled for destruction on $destroyOn (now: ${time})"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
'';
|
||||
};
|
||||
}))
|
||||
]))
|
||||
flatten
|
||||
mkMerge
|
||||
];
|
||||
|
||||
systemd.targets = mapAttrs' (provider: providerConfig: {
|
||||
name = "incandescence-${provider}";
|
||||
value = {
|
||||
description = "An Incandescence | ${provider}";
|
||||
inherit (providerConfig) wantedBy partOf;
|
||||
};
|
||||
}) cfg.providers;
|
||||
|
||||
services.locksmith.providers = mapAttrs (provider: providerConfig: {
|
||||
wantedBy = [ "incandescence-${provider}.target" ];
|
||||
after = [ "incandescence-${provider}.target" ];
|
||||
}) (filterAttrs (_: providerConfig: providerConfig.locksmith) cfg.providers);
|
||||
|
||||
system.ascensions = mapAttrs' (provider: providerConfig: {
|
||||
name = "incandescence-${provider}";
|
||||
value = {
|
||||
distributed = true;
|
||||
requiredBy = map (formula: "ignite-${provider}-${formula}-create.service") (lib.attrNames providerConfig.formulae);
|
||||
before = map (formula: "ignite-${provider}-${formula}-create.service") (lib.attrNames providerConfig.formulae);
|
||||
incantations = lib.mkDefault (i: []);
|
||||
};
|
||||
}) cfg.providers;
|
||||
}
|
|
@ -14,5 +14,6 @@
|
|||
./provider.nix
|
||||
];
|
||||
};
|
||||
simulacrum.deps = [ "chant" "consul" ];
|
||||
};
|
||||
}
|
||||
|
|
|
@ -28,6 +28,10 @@ in
|
|||
command = mkOption {
|
||||
type = types.coercedTo types.package (package: "${package}") types.str;
|
||||
};
|
||||
checkUpdate = mkOption {
|
||||
type = types.coercedTo types.package (package: "${package}") types.str;
|
||||
default = "true";
|
||||
};
|
||||
owner = mkOption {
|
||||
type = types.str;
|
||||
default = "root";
|
||||
|
@ -72,20 +76,27 @@ in
|
|||
activeNodes = lib.unique (lib.flatten (lib.mapAttrsToList (_: secret: secret.nodes) activeSecrets));
|
||||
secretNames = map (name: "${providerRoot}-${name}/") (lib.attrNames activeSecrets);
|
||||
|
||||
createSecret = { path, nodes, owner, mode, group, command }: ''
|
||||
consul kv put ${lib.escapeShellArg path}/mode ${lib.escapeShellArg mode}
|
||||
consul kv put ${lib.escapeShellArg path}/owner ${lib.escapeShellArg owner}
|
||||
consul kv put ${lib.escapeShellArg path}/group ${lib.escapeShellArg group}
|
||||
${lib.concatStringsSep "\n" (map (node: ''
|
||||
consul kv put ${lib.escapeShellArg path}/recipient/${node} "$( (${command}) | age --encrypt --armor -r ${lib.escapeShellArg depot.hours.${node}.ssh.id.publicKey})"
|
||||
'') nodes)}
|
||||
createSecret = { path, nodes, owner, mode, group, command, checkUpdate }: ''
|
||||
if (${checkUpdate}); then
|
||||
consul kv put ${lib.escapeShellArg path}/mode ${lib.escapeShellArg mode}
|
||||
consul kv put ${lib.escapeShellArg path}/owner ${lib.escapeShellArg owner}
|
||||
consul kv put ${lib.escapeShellArg path}/group ${lib.escapeShellArg group}
|
||||
secret="$(mktemp -ut)"
|
||||
(${command}) > "$secret"
|
||||
${lib.concatStringsSep "\n" (map (node: ''
|
||||
consul kv put ${lib.escapeShellArg path}/recipient/${node} "$(age < "$secret" --encrypt --armor -r ${lib.escapeShellArg depot.hours.${node}.ssh.id.publicKey})"
|
||||
'') nodes)}
|
||||
else
|
||||
echo Skipping update for ${lib.escapeShellArg path}
|
||||
fi
|
||||
'';
|
||||
in ''
|
||||
# create/update secrets
|
||||
umask 77
|
||||
${lib.pipe activeSecrets [
|
||||
(lib.mapAttrsToList (secretName: secretConfig: createSecret {
|
||||
path = "${providerRoot}-${secretName}";
|
||||
inherit (secretConfig) nodes mode owner group command;
|
||||
inherit (secretConfig) nodes mode owner group command checkUpdate;
|
||||
}))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
|
|
@ -72,7 +72,7 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
garage = {
|
||||
garage = config.lib.forService "monitoring" {
|
||||
keys = {
|
||||
loki-ingest.locksmith = {
|
||||
nodes = config.services.monitoring.nodes.logging;
|
||||
|
|
91
cluster/services/patroni/create-databases.nix
Normal file
91
cluster/services/patroni/create-databases.nix
Normal file
|
@ -0,0 +1,91 @@
|
|||
{ cluster, config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster.config.services.patroni) secrets;
|
||||
|
||||
patroni = cluster.config.links.patroni-pg-access;
|
||||
|
||||
cfg = cluster.config.patroni;
|
||||
|
||||
writeQueryFile = pkgs.writeText "patroni-query.sql";
|
||||
|
||||
psqlRunFile = file: ''
|
||||
export PGPASSWORD="$(< ${secrets.PATRONI_SUPERUSER_PASSWORD.path})"
|
||||
while ! ${config.services.patroni.postgresqlPackage}/bin/psql 'host=${patroni.ipv4} port=${patroni.portStr} dbname=postgres user=postgres' --tuples-only --csv --file="${file}"; do
|
||||
sleep 3
|
||||
done
|
||||
'';
|
||||
|
||||
psql = query: psqlRunFile (writeQueryFile query);
|
||||
|
||||
psqlSecret = getSecret: queryTemplate: let
|
||||
queryTemplateFile = writeQueryFile queryTemplate;
|
||||
in ''
|
||||
umask 77
|
||||
secretFile="$(mktemp -ut patroniSecret.XXXXXXXXXXXXXXXX)"
|
||||
queryFile="$(mktemp -ut patroniQuery.XXXXXXXXXXXXXXXX)"
|
||||
trap "rm -f $secretFile $queryFile" EXIT
|
||||
${getSecret} > "$secretFile"
|
||||
cp --no-preserve=mode ${queryTemplateFile} "$queryFile"
|
||||
${pkgs.replace-secret}/bin/replace-secret '@SECRET@' "$secretFile" "$queryFile"
|
||||
${psqlRunFile "$queryFile"}
|
||||
'';
|
||||
|
||||
genPassword = pkgs.writeShellScript "patroni-generate-user-password" ''
|
||||
umask 77
|
||||
base64 -w0 /dev/urandom | tr -d /+ | head -c256 | tee "/run/keys/locksmith-provider-patroni-$1"
|
||||
'';
|
||||
in
|
||||
|
||||
{
|
||||
services.incandescence.providers.patroni = lib.mkIf config.services.haproxy.enable {
|
||||
locksmith = true;
|
||||
wantedBy = [ "patroni.service" "multi-user.target" ];
|
||||
partOf = [ "patroni.service" ];
|
||||
wants = [ "postgresql.service" ];
|
||||
after = [ "postgresql.service" ];
|
||||
|
||||
formulae = {
|
||||
user = {
|
||||
destroyAfterDays = 0;
|
||||
create = user: psqlSecret "${genPassword} ${user}" ''
|
||||
CREATE USER ${user} PASSWORD '@SECRET@';
|
||||
'';
|
||||
destroy = psqlSecret "printenv OBJECT" ''
|
||||
DROP USER @SECRET@;
|
||||
'';
|
||||
};
|
||||
database = {
|
||||
destroyAfterDays = 30;
|
||||
deps = [ "user" ];
|
||||
create = db: psql ''
|
||||
CREATE DATABASE ${db} OWNER ${cfg.databases.${db}.owner};
|
||||
'';
|
||||
destroy = psqlSecret "printenv OBJECT" ''
|
||||
DROP DATABASE @SECRET@;
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.locksmith.providers.patroni = lib.mkIf config.services.haproxy.enable {
|
||||
secrets = lib.mapAttrs (user: userConfig: {
|
||||
command = {
|
||||
envFile = ''
|
||||
echo "PGPASSWORD=$(cat /run/keys/locksmith-provider-patroni-${user})"
|
||||
rm -f /run/keys/locksmith-provider-patroni-${user}
|
||||
'';
|
||||
pgpass = ''
|
||||
echo "*:*:*:${user}:$(cat /run/keys/locksmith-provider-patroni-${user})"
|
||||
rm -f /run/keys/locksmith-provider-patroni-${user}
|
||||
'';
|
||||
raw = ''
|
||||
cat /run/keys/locksmith-provider-patroni-${user}
|
||||
rm -f /run/keys/locksmith-provider-patroni-${user}
|
||||
'';
|
||||
}.${userConfig.locksmith.format};
|
||||
checkUpdate = "test -e /run/keys/locksmith-provider-patroni-${user}";
|
||||
inherit (userConfig.locksmith) nodes;
|
||||
}) cfg.users;
|
||||
};
|
||||
}
|
|
@ -1,6 +1,11 @@
|
|||
{ config, lib, ... }:
|
||||
{ config, ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
./options.nix
|
||||
./incandescence.nix
|
||||
];
|
||||
|
||||
links = {
|
||||
patroni-pg-internal.ipv4 = "0.0.0.0";
|
||||
patroni-api.ipv4 = "0.0.0.0";
|
||||
|
@ -15,6 +20,7 @@
|
|||
worker = [
|
||||
./worker.nix
|
||||
./metrics.nix
|
||||
./create-databases.nix
|
||||
];
|
||||
haproxy = ./haproxy.nix;
|
||||
};
|
||||
|
@ -30,5 +36,6 @@
|
|||
PATRONI_REWIND_PASSWORD = default;
|
||||
metricsCredentials.nodes = nodes.worker;
|
||||
};
|
||||
simulacrum.deps = [ "consul" "incandescence" "locksmith" ];
|
||||
};
|
||||
}
|
||||
|
|
10
cluster/services/patroni/incandescence.nix
Normal file
10
cluster/services/patroni/incandescence.nix
Normal file
|
@ -0,0 +1,10 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
incandescence.providers.patroni = {
|
||||
objects = {
|
||||
user = lib.attrNames config.patroni.users;
|
||||
database = lib.attrNames config.patroni.databases;
|
||||
};
|
||||
};
|
||||
}
|
37
cluster/services/patroni/options.nix
Normal file
37
cluster/services/patroni/options.nix
Normal file
|
@ -0,0 +1,37 @@
|
|||
{ lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) mkOption;
|
||||
inherit (lib.types) attrsOf enum listOf submodule str;
|
||||
in
|
||||
|
||||
{
|
||||
options.patroni = {
|
||||
databases = mkOption {
|
||||
type = attrsOf (submodule ({ name, ... }: {
|
||||
options = {
|
||||
owner = mkOption {
|
||||
type = str;
|
||||
default = name;
|
||||
};
|
||||
};
|
||||
}));
|
||||
};
|
||||
users = mkOption {
|
||||
type = attrsOf (submodule ({ ... }: {
|
||||
options = {
|
||||
locksmith = {
|
||||
nodes = mkOption {
|
||||
type = listOf str;
|
||||
default = [];
|
||||
};
|
||||
format = mkOption {
|
||||
type = enum [ "pgpass" "envFile" "raw" ];
|
||||
default = "pgpass";
|
||||
};
|
||||
};
|
||||
};
|
||||
}));
|
||||
};
|
||||
};
|
||||
}
|
|
@ -7,6 +7,8 @@ in
|
|||
{
|
||||
imports = [
|
||||
./options.nix
|
||||
./incandescence.nix
|
||||
./simulacrum/test-data.nix
|
||||
];
|
||||
|
||||
services.storage = {
|
||||
|
@ -30,11 +32,16 @@ in
|
|||
heresy = [
|
||||
./heresy.nix
|
||||
./s3ql-upgrades.nix
|
||||
] ++ lib.optionals config.simulacrum [
|
||||
./simulacrum/snakeoil-heresy-passphrase.nix
|
||||
];
|
||||
garage = [
|
||||
./garage.nix
|
||||
./garage-options.nix
|
||||
./garage-layout.nix
|
||||
./incandescence-ascensions.nix
|
||||
] ++ lib.optionals config.simulacrum [
|
||||
./simulacrum/snakeoil-rpc-secret.nix
|
||||
];
|
||||
garageConfig = [
|
||||
./garage-gateway.nix
|
||||
|
@ -48,6 +55,11 @@ in
|
|||
garageInternal = [ ./garage-internal.nix ];
|
||||
garageExternal = [ ./garage-external.nix ];
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "consul" "locksmith" "incandescence" "patroni" "ways" ];
|
||||
settings = ./test.nix;
|
||||
};
|
||||
};
|
||||
|
||||
links = {
|
||||
|
@ -86,7 +98,10 @@ in
|
|||
};
|
||||
|
||||
garage = {
|
||||
keys.storage-prophet = {};
|
||||
keys.storage-prophet.locksmith = {
|
||||
nodes = [ "prophet" ];
|
||||
format = "s3ql";
|
||||
};
|
||||
buckets.storage-prophet = {
|
||||
allow.storage-prophet = [ "read" "write" ];
|
||||
};
|
||||
|
|
|
@ -8,7 +8,7 @@ in
|
|||
services.external-storage = {
|
||||
fileSystems.external = {
|
||||
mountpoint = "/srv/storage";
|
||||
authFile = ./secrets/external-storage-auth-${hostName}.age;
|
||||
locksmithSecret = "garage-storage-${hostName}";
|
||||
backend = "s3c4://${cluster.config.links.garageS3.hostname}/storage-${hostName}";
|
||||
backendOptions = [ "disable-expect100" ];
|
||||
};
|
||||
|
|
|
@ -26,62 +26,6 @@ let
|
|||
sleep 1
|
||||
done
|
||||
}
|
||||
# FIXME: returns bogus empty string when one of the lists is empty
|
||||
diffAdded() {
|
||||
comm -13 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
|
||||
}
|
||||
diffRemoved() {
|
||||
comm -23 <(printf '%s\n' $1 | sort) <(printf '%s\n' $2 | sort)
|
||||
}
|
||||
# FIXME: this does not handle list items with spaces
|
||||
listKeys() {
|
||||
garage key list | tail -n +2 | grep -ow '[^ ]*$' || true
|
||||
}
|
||||
ensureKeys() {
|
||||
old="$(listKeys)"
|
||||
if [[ -z "$1" ]]; then
|
||||
for key in $old; do
|
||||
garage key delete --yes "$key"
|
||||
done
|
||||
elif [[ -z "$old" ]]; then
|
||||
for key in $1; do
|
||||
# don't print secret key
|
||||
garage key new --name "$key" >/dev/null
|
||||
echo Key "$key" was created.
|
||||
done
|
||||
else
|
||||
diffAdded "$old" "$1" | while read key; do
|
||||
# don't print secret key
|
||||
garage key new --name "$key" >/dev/null
|
||||
echo Key "$key" was created.
|
||||
done
|
||||
diffRemoved "$old" "$1" | while read key; do
|
||||
garage key delete --yes "$key"
|
||||
done
|
||||
fi
|
||||
}
|
||||
listBuckets() {
|
||||
garage bucket list | tail -n +2 | grep -ow '^ *[^ ]*' | tr -d ' ' || true
|
||||
}
|
||||
ensureBuckets() {
|
||||
old="$(listBuckets)"
|
||||
if [[ -z "$1" ]]; then
|
||||
for bucket in $old; do
|
||||
garage bucket delete --yes "$bucket"
|
||||
done
|
||||
elif [[ -z "$old" ]]; then
|
||||
for bucket in $1; do
|
||||
garage bucket create "$bucket"
|
||||
done
|
||||
else
|
||||
diffAdded "$old" "$1" | while read bucket; do
|
||||
garage bucket create "$bucket"
|
||||
done
|
||||
diffRemoved "$old" "$1" | while read bucket; do
|
||||
garage bucket delete --yes "$bucket"
|
||||
done
|
||||
fi
|
||||
}
|
||||
'';
|
||||
in
|
||||
|
||||
|
@ -118,7 +62,7 @@ in
|
|||
};
|
||||
format = mkOption {
|
||||
description = "Locksmith secret format.";
|
||||
type = enum [ "files" "aws" "envFile" ];
|
||||
type = enum [ "files" "aws" "envFile" "s3ql" ];
|
||||
default = "files";
|
||||
};
|
||||
owner = mkOption {
|
||||
|
@ -203,9 +147,7 @@ in
|
|||
garage layout apply --version 1
|
||||
'';
|
||||
};
|
||||
garage-apply = {
|
||||
distributed.enable = true;
|
||||
wantedBy = [ "garage.service" "multi-user.target" ];
|
||||
garage-ready = {
|
||||
wants = [ "garage.service" ];
|
||||
after = [ "garage.service" "garage-layout-init.service" ];
|
||||
path = [ config.services.garage.package ];
|
||||
|
@ -219,54 +161,70 @@ in
|
|||
script = ''
|
||||
source ${garageShellLibrary}
|
||||
waitForGarageOperational
|
||||
|
||||
ensureKeys '${lib.concatStringsSep " " (lib.attrNames cfg.keys)}'
|
||||
ensureBuckets '${lib.concatStringsSep " " (lib.attrNames cfg.buckets)}'
|
||||
|
||||
# key permissions
|
||||
${lib.pipe cfg.keys [
|
||||
(lib.mapAttrsToList (key: kCfg: ''
|
||||
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} '${key}' --create-bucket >/dev/null
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket permissions
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg:
|
||||
lib.mapAttrsToList (key: perms: ''
|
||||
garage bucket allow '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") perms)}
|
||||
garage bucket deny '${bucket}' --key '${key}' ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
|
||||
'') bCfg.allow
|
||||
))
|
||||
lib.flatten
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket quotas
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg: ''
|
||||
garage bucket set-quotas '${bucket}' \
|
||||
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
|
||||
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
# bucket website access
|
||||
${lib.pipe cfg.buckets [
|
||||
(lib.mapAttrsToList (bucket: bCfg: ''
|
||||
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} '${bucket}'
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
services.incandescence.providers.garage = {
|
||||
locksmith = true;
|
||||
wantedBy = [ "garage.service" "multi-user.target" ];
|
||||
partOf = [ "garage.service" ];
|
||||
wants = [ "garage-ready.service" ];
|
||||
after = [ "garage-ready.service" ];
|
||||
|
||||
packages = [
|
||||
config.services.garage.package
|
||||
];
|
||||
formulae = {
|
||||
key = {
|
||||
destroyAfterDays = 0;
|
||||
create = key: ''
|
||||
# don't print secret key
|
||||
garage key new --name ${lib.escapeShellArg key} >/dev/null
|
||||
echo Key ${lib.escapeShellArg key} was created.
|
||||
'';
|
||||
destroy = ''
|
||||
garage key delete --yes "$OBJECT"
|
||||
'';
|
||||
change = key: let
|
||||
kCfg = cfg.keys.${key};
|
||||
in ''
|
||||
garage key ${if kCfg.allow.createBucket then "allow" else "deny"} ${lib.escapeShellArg key} --create-bucket >/dev/null
|
||||
'';
|
||||
};
|
||||
bucket = {
|
||||
deps = [ "key" ];
|
||||
destroyAfterDays = 30;
|
||||
create = bucket: ''
|
||||
garage bucket create ${lib.escapeShellArg bucket}
|
||||
'';
|
||||
destroy = ''
|
||||
garage bucket delete --yes "$OBJECT"
|
||||
'';
|
||||
change = bucket: let
|
||||
bCfg = cfg.buckets.${bucket};
|
||||
in ''
|
||||
# permissions
|
||||
${lib.concatStringsSep "\n" (lib.flatten (
|
||||
lib.mapAttrsToList (key: perms: ''
|
||||
garage bucket allow ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") perms)}
|
||||
garage bucket deny ${lib.escapeShellArg bucket} --key ${lib.escapeShellArg key} ${lib.escapeShellArgs (map (x: "--${x}") (lib.subtractLists perms [ "read" "write" "owner" ]))}
|
||||
'') bCfg.allow
|
||||
))}
|
||||
|
||||
# quotas
|
||||
garage bucket set-quotas ${lib.escapeShellArg bucket} \
|
||||
--max-objects '${if bCfg.quotas.maxObjects == null then "none" else toString bCfg.quotas.maxObjects}' \
|
||||
--max-size '${if bCfg.quotas.maxSize == null then "none" else toString bCfg.quotas.maxSize}'
|
||||
|
||||
# website access
|
||||
garage bucket website ${if bCfg.web.enable then "--allow" else "--deny"} ${lib.escapeShellArg bucket}
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.locksmith.providers.garage = {
|
||||
wantedBy = [ "garage-apply.service" ];
|
||||
after = [ "garage-apply.service" ];
|
||||
secrets = lib.mkMerge (lib.mapAttrsToList (key: kCfg: let
|
||||
common = {
|
||||
inherit (kCfg.locksmith) mode owner group nodes;
|
||||
|
@ -291,6 +249,12 @@ in
|
|||
AWS_ACCESS_KEY_ID=@@GARAGE_KEY_ID@@
|
||||
AWS_SECRET_ACCESS_KEY=@@GARAGE_SECRET_KEY@@
|
||||
'';
|
||||
s3ql = ''
|
||||
[s3c]
|
||||
storage-url: s3c4://
|
||||
backend-login: @@GARAGE_KEY_ID@@
|
||||
backend-password: @@GARAGE_SECRET_KEY@@
|
||||
'';
|
||||
}.${kCfg.locksmith.format};
|
||||
in {
|
||||
${key} = common // {
|
||||
|
|
|
@ -63,6 +63,8 @@ in
|
|||
};
|
||||
|
||||
systemd.services.garage = {
|
||||
requires = [ "consul-ready.service" ];
|
||||
after = [ "consul-ready.service" ];
|
||||
unitConfig = {
|
||||
RequiresMountsFor = [ cfg.settings.data_dir ];
|
||||
};
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
unitDescription = "Heresy Filesystem";
|
||||
authFile = ./secrets/heresy-encryption-key.age;
|
||||
underlay = "heresy";
|
||||
encrypt = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
18
cluster/services/storage/incandescence-ascensions.nix
Normal file
18
cluster/services/storage/incandescence-ascensions.nix
Normal file
|
@ -0,0 +1,18 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
system.ascensions = {
|
||||
incandescence-garage = lib.mkIf (config.services.incandescence.providers ? garage) {
|
||||
incantations = i: [
|
||||
(i.runGarage /*bash*/ ''
|
||||
garage bucket list | tail -n +2 | cut -d' ' -f3 | while read bucket; do
|
||||
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/bucket/$1/alive" true''} "$bucket"
|
||||
done
|
||||
garage key list | tail -n +2 | cut -d' ' -f5 | while read key; do
|
||||
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/key/$1/alive" true''} "$key"
|
||||
done
|
||||
'')
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
10
cluster/services/storage/incandescence.nix
Normal file
10
cluster/services/storage/incandescence.nix
Normal file
|
@ -0,0 +1,10 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
incandescence.providers.garage = {
|
||||
objects = {
|
||||
key = lib.attrNames config.garage.keys;
|
||||
bucket = lib.attrNames config.garage.buckets;
|
||||
};
|
||||
};
|
||||
}
|
|
@ -1,11 +0,0 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 NO562A tC8lfwNJIXjVJImBq25v/NGIQ1Ns24NpCzksbw/eb3w
|
||||
2hQltUYSO2Gpjd+49IQR1UJOhy33xWvNH6dx+uGDvFA
|
||||
-> ssh-ed25519 5/zT0w dapxQ/VV0peQKMwghQJ91wQVahYOqxw2QrXqQCau82c
|
||||
0DnIF5ISoB5htYA3X5DSTgLJXLSkqjz1O0CMcmnnrjQ
|
||||
-> ssh-ed25519 YIaSKQ ehv+WWCLC/co9lhpa+cAdqJUG33L/Vkn6lUXOwNRV2w
|
||||
LEobbvvpq6lPNbzasGeXf9NabN150ZVe5n5OJNgbyD4
|
||||
--- FrT2CFmuWQ+vKGbBY2pGT90Mu8WzXfpbIAzYdR3Vb2w
|
||||
™ªg¬NÑ8´¨\K!p
«ï…7ù¶›käõ¯#ŒÏu›µ*{}Tþ0·|@ÉÿàE>z„'-RxK¸zB£ÿä©n*0¢÷~OVû®4¦qûÁ]^(ìì>-‡3ÌÙe0aí<61>¥ì.oòÙC)†‡4g¶ð»7NzÉ”ºnÒÃî®Mª†x6àöãö×'[Ô6ãw?ÿª€ãi=†vèEJˆB
|
||||
µÿÂ9gÏi"Q
–ÿ
|
||||
™›Ù®à
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
environment.etc."dummy-secrets/storageAuth-heresy".text = ''
|
||||
[local]
|
||||
storage-url: local://
|
||||
fs-passphrase: simulacrum
|
||||
'';
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
}
|
8
cluster/services/storage/simulacrum/test-data.nix
Normal file
8
cluster/services/storage/simulacrum/test-data.nix
Normal file
|
@ -0,0 +1,8 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
garage = lib.mkIf config.simulacrum {
|
||||
keys.testkey = {};
|
||||
buckets.testbucket.allow.testKey = [ "read" "write" ];
|
||||
};
|
||||
}
|
105
cluster/services/storage/test.nix
Normal file
105
cluster/services/storage/test.nix
Normal file
|
@ -0,0 +1,105 @@
|
|||
{ cluster, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster.config.services.storage) nodes;
|
||||
|
||||
firstGarageNode = lib.elemAt nodes.garage 0;
|
||||
in
|
||||
|
||||
{
|
||||
nodes = lib.genAttrs nodes.garage (node: {
|
||||
services.garage = {
|
||||
layout.initial = lib.genAttrs nodes.garage (_: {
|
||||
capacity = lib.mkOverride 51 1000;
|
||||
});
|
||||
};
|
||||
specialisation.modifiedLayout = {
|
||||
inheritParentConfig = true;
|
||||
configuration = {
|
||||
services.garage = {
|
||||
layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
|
||||
(i.runGarage ''
|
||||
garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
|
||||
garage layout apply --version 2
|
||||
'')
|
||||
]);
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
|
||||
garage1 = nodes[0]
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap new cluster"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
with subtest("should apply new layout with ascension"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
|
||||
|
||||
for node in nodes:
|
||||
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
consulConfig = json.loads(garage1.succeed("cat /etc/consul.json"))
|
||||
addr = consulConfig["addresses"]["http"]
|
||||
port = consulConfig["ports"]["http"]
|
||||
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
|
||||
with subtest("should apply new layout from scratch"):
|
||||
for node in nodes:
|
||||
node.systemctl("stop garage.service")
|
||||
node.succeed("rm -rf /var/lib/garage-metadata")
|
||||
garage1.succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/garage")
|
||||
|
||||
for node in nodes:
|
||||
node.systemctl("start garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "${toString ((lib.length nodes.garage) - 1)}" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should create specified buckets and keys"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("incandescence-garage.target")
|
||||
garage1.succeed("garage key list | grep testkey")
|
||||
garage1.succeed("garage bucket list | grep testbucket")
|
||||
|
||||
with subtest("should delete unspecified keys"):
|
||||
garage1.succeed("garage bucket create unwantedbucket")
|
||||
garage1.succeed("garage key new --name unwantedkey")
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/key/unwantedkey/alive true")
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/alive true")
|
||||
garage1.succeed("systemctl restart garage.service")
|
||||
garage1.wait_for_unit("incandescence-garage.target")
|
||||
garage1.fail("garage key list | grep unwantedkey")
|
||||
garage1.succeed("garage bucket list | grep unwantedbucket")
|
||||
|
||||
with subtest("should delete unspecified buckets after grace period"):
|
||||
garage1.succeed(f"{setEnv} consul kv put services/incandescence/providers/garage/formulae/bucket/unwantedbucket/destroyOn 1")
|
||||
garage1.succeed("systemctl restart garage.service")
|
||||
garage1.wait_for_unit("incandescence-garage.target")
|
||||
garage1.fail("garage bucket list | grep unwantedbucket")
|
||||
'';
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@
|
|||
services.ways = {
|
||||
nodes.host = config.services.websites.nodes.host;
|
||||
nixos.host = ./host.nix;
|
||||
simulacrum.deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
|
||||
};
|
||||
|
||||
dns.records = lib.mapAttrs'
|
||||
|
|
|
@ -10,6 +10,19 @@ let
|
|||
};
|
||||
|
||||
getExtAddr = host: host.interfaces.primary.addrPublic;
|
||||
|
||||
snakeoilPublicKeys = {
|
||||
checkmate = "TESTtbFybW5YREwtd18a1A4StS4YAIUS5/M1Lv0jHjA=";
|
||||
grail = "TEsTh7bthkaDh9A1CpqDi/F121ao5lRZqIJznLH8mB4=";
|
||||
thunderskin = "tEST6afFmVN18o+EiWNFx+ax3MJwdQIeNfJSGEpffXw=";
|
||||
VEGAS = "tEsT6s7VtM5C20eJBaq6UlQydAha8ATlmrTRe9T5jnM=";
|
||||
prophet = "TEstYyb5IoqSL53HbSQwMhTaR16sxcWcMmXIBPd+1gE=";
|
||||
};
|
||||
|
||||
grease = hourName: realPublicKey: if config.simulacrum then
|
||||
snakeoilPublicKeys.${hourName}
|
||||
else
|
||||
realPublicKey;
|
||||
in
|
||||
{
|
||||
vars = {
|
||||
|
@ -22,7 +35,7 @@ in
|
|||
extra = {
|
||||
meshIp = "10.1.1.32";
|
||||
inherit meshNet;
|
||||
pubKey = "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
|
||||
pubKey = grease "checkmate" "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
|
||||
extraRoutes = [];
|
||||
};
|
||||
};
|
||||
|
@ -31,7 +44,7 @@ in
|
|||
extra = {
|
||||
meshIp = "10.1.1.6";
|
||||
inherit meshNet;
|
||||
pubKey = "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
|
||||
pubKey = grease "grail" "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
|
||||
extraRoutes = [];
|
||||
};
|
||||
};
|
||||
|
@ -40,7 +53,7 @@ in
|
|||
extra = {
|
||||
meshIp = "10.1.1.4";
|
||||
inherit meshNet;
|
||||
pubKey = "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
|
||||
pubKey = grease "thunderskin" "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
|
||||
extraRoutes = [];
|
||||
};
|
||||
};
|
||||
|
@ -49,7 +62,7 @@ in
|
|||
extra = {
|
||||
meshIp = "10.1.1.5";
|
||||
inherit meshNet;
|
||||
pubKey = "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
|
||||
pubKey = grease "VEGAS" "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
|
||||
extraRoutes = [ "${hours.VEGAS.interfaces.vstub.addr}/32" "10.10.0.0/16" ];
|
||||
};
|
||||
};
|
||||
|
@ -58,7 +71,7 @@ in
|
|||
extra = {
|
||||
meshIp = "10.1.1.9";
|
||||
inherit meshNet;
|
||||
pubKey = "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
|
||||
pubKey = grease "prophet" "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
|
||||
extraRoutes = [];
|
||||
};
|
||||
};
|
||||
|
@ -69,8 +82,12 @@ in
|
|||
storm = [ "VEGAS" ];
|
||||
};
|
||||
nixos = {
|
||||
mesh = ./mesh.nix;
|
||||
storm = ./storm.nix;
|
||||
mesh = [
|
||||
./mesh.nix
|
||||
] ++ lib.optionals config.simulacrum [
|
||||
./simulacrum/snakeoil-keys.nix
|
||||
];
|
||||
storm = [ ./storm.nix ];
|
||||
};
|
||||
secrets.meshPrivateKey = {
|
||||
nodes = config.services.wireguard.nodes.mesh;
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
MNvWpMluuzQvPyGTp7jtyPSyz6n9lIly/WX1gW2NAHg=
|
|
@ -0,0 +1 @@
|
|||
YHzP8rBP6qiXs6ZdnvHop9KnCYRADIEejwZzAzvj8m4=
|
|
@ -0,0 +1 @@
|
|||
uD7X5E6N9d0sN+xPr/bWnehSa3bAok741GO7Z4I+Z3I=
|
|
@ -0,0 +1 @@
|
|||
QHyIJ3HoKGGFN28qOrQP4UyoQMP5bM7Idn2MzayKzEM=
|
|
@ -0,0 +1 @@
|
|||
YLl+hkWaCWx/5PpWs3cQ+bKqYdJef/qZ+FMTsM9ammM=
|
6
cluster/services/wireguard/simulacrum/snakeoil-keys.nix
Normal file
6
cluster/services/wireguard/simulacrum/snakeoil-keys.nix
Normal file
|
@ -0,0 +1,6 @@
|
|||
{ lib, config, ... }: {
|
||||
config.environment.etc = {
|
||||
"dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
|
||||
"dummy-secrets/wireguard-key-storm".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
|
||||
};
|
||||
}
|
16
cluster/simulacrum/checks.nix
Normal file
16
cluster/simulacrum/checks.nix
Normal file
|
@ -0,0 +1,16 @@
|
|||
{ config, extendModules, lib, ... }:
|
||||
|
||||
{
|
||||
perSystem = { pkgs, system, ... }: {
|
||||
checks = lib.mkIf (system == "x86_64-linux") (lib.mapAttrs' (name: svc: let
|
||||
runSimulacrum = pkgs.callPackage ./. {
|
||||
inherit config extendModules;
|
||||
};
|
||||
in {
|
||||
name = "simulacrum-${name}";
|
||||
value = runSimulacrum {
|
||||
service = name;
|
||||
};
|
||||
}) (lib.filterAttrs (_: svc: svc.simulacrum.enable) config.cluster.config.services));
|
||||
};
|
||||
}
|
134
cluster/simulacrum/default.nix
Normal file
134
cluster/simulacrum/default.nix
Normal file
|
@ -0,0 +1,134 @@
|
|||
{ testers, config, extendModules, lib, system }:
|
||||
|
||||
{ service }:
|
||||
|
||||
let
|
||||
serviceConfig = config.cluster.config.services.${service};
|
||||
serviceList = getDepsRecursive [] service;
|
||||
allAugments = map (svc: config.cluster.config.services.${svc}.simulacrum.augments) serviceList;
|
||||
|
||||
getDepsRecursive = acc: service: let
|
||||
deps = lib.subtractLists acc config.cluster.config.services.${service}.simulacrum.deps;
|
||||
acc' = acc ++ [ service ];
|
||||
recurse = getDepsRecursive acc';
|
||||
in lib.unique (lib.flatten ([ service ] ++ map recurse deps));
|
||||
|
||||
lift = config;
|
||||
|
||||
snakeoil = {
|
||||
ssh = {
|
||||
public = lib.fileContents ../../packages/checks/snakeoil/ssh/snakeoil-key.pub;
|
||||
private = ../../packages/checks/snakeoil/ssh/snakeoil-key;
|
||||
};
|
||||
};
|
||||
|
||||
nodes = lib.attrNames config.gods.fromLight;
|
||||
nodes' = lib.attrNames (config.gods.fromLight // { nowhere = null; });
|
||||
digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes' (lib.range 1 255));
|
||||
depot' = extendModules {
|
||||
modules = [
|
||||
({ config, ... }: {
|
||||
gods.fromLight = lib.mapAttrs (name: cfg: {
|
||||
interfaces.primary = {
|
||||
link = lib.mkForce "vprimary";
|
||||
};
|
||||
ssh.id.publicKey = lib.mkForce snakeoil.ssh.public;
|
||||
}) lift.gods.fromLight;
|
||||
|
||||
cluster = lib.mkForce (lift.cluster.extendModules {
|
||||
specialArgs.depot = config;
|
||||
modules = [
|
||||
{
|
||||
simulacrum = true;
|
||||
testConfig = {
|
||||
subject = service;
|
||||
activeServices = serviceList;
|
||||
};
|
||||
}
|
||||
];
|
||||
});
|
||||
})
|
||||
];
|
||||
};
|
||||
specialArgs = depot'.config.lib.summon system lib.id;
|
||||
in
|
||||
|
||||
testers.runNixOSTest {
|
||||
name = "simulacrum-${service}";
|
||||
|
||||
imports = [
|
||||
serviceConfig.simulacrum.settings
|
||||
./nowhere
|
||||
{
|
||||
nodes.nowhere.imports = [
|
||||
config.flake.nixosModules.port-magic
|
||||
];
|
||||
}
|
||||
] ++ allAugments;
|
||||
|
||||
_module.args = {
|
||||
inherit (depot'.config) cluster;
|
||||
};
|
||||
|
||||
node = { inherit specialArgs; };
|
||||
nodes = lib.genAttrs nodes (node: let
|
||||
hour = depot'.config.hours.${node};
|
||||
in {
|
||||
imports = [
|
||||
specialArgs.depot.hours.${node}.nixos
|
||||
../../packages/checks/modules/nixos/age-dummy-secrets
|
||||
../../packages/checks/modules/nixos/external-storage.nix
|
||||
] ++ depot'.config.cluster.config.out.injectNixosConfigForServices serviceList node;
|
||||
|
||||
boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
|
||||
networking = {
|
||||
interfaces = {
|
||||
${hour.interfaces.primary.link} = {
|
||||
useDHCP = lib.mkForce false;
|
||||
virtual = true;
|
||||
ipv4.addresses = lib.mkForce ([
|
||||
{
|
||||
address = hour.interfaces.primary.addr;
|
||||
prefixLength = 32;
|
||||
}
|
||||
] ++ lib.optional hour.interfaces.primary.isNat {
|
||||
address = hour.interfaces.primary.addrPublic;
|
||||
prefixLength = 32;
|
||||
});
|
||||
};
|
||||
eth1.ipv4.routes = lib.pipe nodes [
|
||||
(lib.filter (n: n != node))
|
||||
(map (n: let
|
||||
hour = depot'.config.hours.${n};
|
||||
in {
|
||||
address = hour.interfaces.primary.addrPublic;
|
||||
prefixLength = 32;
|
||||
via = "192.168.1.${toString digits.${n}}";
|
||||
}))
|
||||
];
|
||||
};
|
||||
|
||||
firewall.extraCommands = lib.mkAfter (lib.optionalString (hour.interfaces.primary.isNat) ''
|
||||
# self-nat
|
||||
iptables -t nat -A PREROUTING -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
|
||||
iptables -t nat -A OUTPUT -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
|
||||
iptables -t nat -A POSTROUTING -s ${hour.interfaces.primary.addr} -j SNAT --to-source ${hour.interfaces.primary.addrPublic}
|
||||
'');
|
||||
};
|
||||
|
||||
systemd.services = {
|
||||
hyprspace.enable = false;
|
||||
};
|
||||
|
||||
environment.etc = {
|
||||
"ssh/ssh_host_ed25519_key" = {
|
||||
source = snakeoil.ssh.private;
|
||||
mode = "0400";
|
||||
};
|
||||
};
|
||||
virtualisation = {
|
||||
cores = 2;
|
||||
memorySize = 4096;
|
||||
};
|
||||
});
|
||||
}
|
101
cluster/simulacrum/nowhere/default.nix
Normal file
101
cluster/simulacrum/nowhere/default.nix
Normal file
|
@ -0,0 +1,101 @@
|
|||
{ cluster, config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
lift = config;
|
||||
|
||||
cfsslConfigIntermediateCA = pkgs.writeText "simulacrum-cfssl-config.json" (builtins.toJSON {
|
||||
signing = {
|
||||
default.expiry = "8760h";
|
||||
profiles.intermediate = {
|
||||
expiry = "8760h";
|
||||
usages = [
|
||||
"cert sign"
|
||||
"crl sign"
|
||||
];
|
||||
ca_constraint = {
|
||||
is_ca = true;
|
||||
max_path_len = 1;
|
||||
};
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
caCsr = pkgs.writeText "simulacrum-ca-csr.json" (builtins.toJSON {
|
||||
CN = "Simulacrum Root CA";
|
||||
});
|
||||
|
||||
ca = pkgs.runCommand "simulacrum-snakeoil-ca" {
|
||||
nativeBuildInputs = [
|
||||
pkgs.cfssl
|
||||
];
|
||||
} ''
|
||||
mkdir $out
|
||||
cfssl gencert --initca ${caCsr} | cfssljson --bare $out/ca
|
||||
'';
|
||||
|
||||
genCert = extraFlags: csrData: let
|
||||
csr = pkgs.writeText "simulacrum-csr.json" (builtins.toJSON csrData);
|
||||
in pkgs.runCommand "simulacrum-snakeoil-cert" {
|
||||
nativeBuildInputs = [
|
||||
pkgs.cfssl
|
||||
];
|
||||
} ''
|
||||
mkdir $out
|
||||
cfssl gencert ${lib.escapeShellArgs ([
|
||||
"--ca=file:${ca}/ca.pem"
|
||||
"--ca-key=file:${ca}/ca-key.pem"
|
||||
] ++ extraFlags ++ [
|
||||
csr
|
||||
])} | cfssljson --bare $out/cert
|
||||
'';
|
||||
|
||||
genHostCert = hostname: genCert [ "--hostname=${hostname}" ] { CN = hostname; };
|
||||
|
||||
getNodeAddr = node: (builtins.head config.nodes.${node}.networking.interfaces.eth1.ipv4.addresses).address;
|
||||
in
|
||||
|
||||
{
|
||||
imports = [
|
||||
./options.nix
|
||||
];
|
||||
defaults = {
|
||||
networking.hosts."${getNodeAddr "nowhere"}" = lib.attrNames config.nowhere.names;
|
||||
security.pki.certificateFiles = [
|
||||
"${ca}/ca.pem"
|
||||
];
|
||||
};
|
||||
|
||||
nowhere.certs = {
|
||||
inherit ca;
|
||||
intermediate = genCert [ "--config=${cfsslConfigIntermediateCA}" "--profile=intermediate" ] {
|
||||
CN = "Simulacrum Intermediate CA";
|
||||
};
|
||||
};
|
||||
|
||||
nodes.nowhere = { config, depot, ... }: {
|
||||
networking = {
|
||||
firewall.allowedTCPPorts = [ 443 ];
|
||||
interfaces.eth1.ipv4.routes = lib.mapAttrsToList (name: hour: {
|
||||
address = hour.interfaces.primary.addrPublic;
|
||||
prefixLength = 32;
|
||||
via = getNodeAddr name;
|
||||
}) depot.gods.fromLight;
|
||||
nameservers = map (name: depot.hours.${name}.interfaces.primary.addrPublic) cluster.config.services.dns.nodes.authoritative;
|
||||
};
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
recommendedProxySettings = true;
|
||||
virtualHosts = lib.mapAttrs (name: link: let
|
||||
cert = genHostCert name;
|
||||
in {
|
||||
forceSSL = true;
|
||||
sslCertificate = "${cert}/cert.pem";
|
||||
sslCertificateKey = "${cert}/cert-key.pem";
|
||||
locations."/" = {
|
||||
proxyPass = config.links.${link}.url;
|
||||
extraConfig = "proxy_ssl_verify off;";
|
||||
};
|
||||
}) lift.nowhere.names;
|
||||
};
|
||||
};
|
||||
}
|
16
cluster/simulacrum/nowhere/options.nix
Normal file
16
cluster/simulacrum/nowhere/options.nix
Normal file
|
@ -0,0 +1,16 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
options.nowhere = {
|
||||
names = lib.mkOption {
|
||||
description = "Hostnames that point Nowhere.";
|
||||
type = with lib.types; attrsOf str;
|
||||
default = {};
|
||||
};
|
||||
certs = lib.mkOption {
|
||||
description = "Snakeoil certificate packages.";
|
||||
type = with lib.types; attrsOf package;
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
}
|
|
@ -8,6 +8,8 @@ let
|
|||
cfgAge = config.age;
|
||||
|
||||
create = lib.flip lib.mapAttrs';
|
||||
|
||||
createFiltered = pred: attrs: f: create (lib.filterAttrs pred attrs) f;
|
||||
in
|
||||
|
||||
{
|
||||
|
@ -20,12 +22,17 @@ in
|
|||
fileSystems = lib.mkOption {
|
||||
description = "S3QL-based filesystems on top of CIFS mountpoints.";
|
||||
default = {};
|
||||
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: {
|
||||
type = with lib.types; lazyAttrsOf (submodule ({ config, name, ... }: let
|
||||
authFile = if config.locksmithSecret != null then
|
||||
"/run/locksmith/${config.locksmithSecret}"
|
||||
else
|
||||
cfgAge.secrets."storageAuth-${name}".path;
|
||||
in {
|
||||
imports = [ ./filesystem-type.nix ];
|
||||
backend = lib.mkIf (config.underlay != null) "local://${cfg.underlays.${config.underlay}.mountpoint}";
|
||||
commonArgs = [
|
||||
"--cachedir" config.cacheDir
|
||||
"--authfile" cfgAge.secrets."storageAuth-${name}".path
|
||||
"--authfile" authFile
|
||||
] ++ (lib.optionals (config.backendOptions != []) [ "--backend-options" (lib.concatStringsSep "," config.backendOptions) ]);
|
||||
}));
|
||||
};
|
||||
|
@ -57,9 +64,14 @@ in
|
|||
|
||||
age.secrets = lib.mkMerge [
|
||||
(create cfg.underlays (name: ul: lib.nameValuePair "cifsCredentials-${name}" { file = ul.credentialsFile; }))
|
||||
(create cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
|
||||
(createFiltered (_: fs: fs.locksmithSecret == null) cfg.fileSystems (name: fs: lib.nameValuePair "storageAuth-${name}" { file = fs.authFile; }))
|
||||
];
|
||||
|
||||
services.locksmith.waitForSecrets = createFiltered (_: fs: fs.locksmithSecret != null) cfg.fileSystems (name: fs: {
|
||||
name = fs.unitName;
|
||||
value = [ fs.locksmithSecret ];
|
||||
});
|
||||
|
||||
fileSystems = create cfg.underlays (name: ul: {
|
||||
name = ul.mountpoint;
|
||||
value = {
|
||||
|
@ -97,7 +109,13 @@ in
|
|||
value = let
|
||||
isUnderlay = fs.underlay != null;
|
||||
|
||||
fsType = if isUnderlay then "local" else lib.head (lib.strings.match "([a-z0-9]*)://.*" fs.backend);
|
||||
backendParts = lib.strings.match "([a-z0-9]*)://([^/]*)/([^/]*)(/.*)?" fs.backend;
|
||||
|
||||
fsType = if isUnderlay then "local" else lib.head backendParts;
|
||||
|
||||
s3Endpoint = assert fsType == "s3c4"; lib.elemAt backendParts 1;
|
||||
|
||||
s3Bucket = assert fsType == "s3c4"; lib.elemAt backendParts 2;
|
||||
|
||||
localBackendPath = if isUnderlay then cfg.underlays.${fs.underlay}.mountpoint else lib.head (lib.strings.match "[a-z0-9]*://(/.*)" fs.backend);
|
||||
in {
|
||||
|
@ -120,8 +138,12 @@ in
|
|||
ExecStartPre = map lib.escapeShellArgs [
|
||||
[
|
||||
(let
|
||||
authFile = if fs.locksmithSecret != null then
|
||||
"/run/locksmith/${fs.locksmithSecret}"
|
||||
else
|
||||
cfgAge.secrets."storageAuth-${name}".path;
|
||||
mkfsEncrypted = ''
|
||||
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${config.age.secrets."storageAuth-${name}".path}' \
|
||||
${pkgs.gnugrep}/bin/grep -m1 fs-passphrase: '${authFile}' \
|
||||
| cut -d' ' -f2- \
|
||||
| ${s3ql}/bin/mkfs.s3ql ${lib.escapeShellArgs fs.commonArgs} -L '${name}' '${fs.backend}'
|
||||
'';
|
||||
|
@ -132,6 +154,11 @@ in
|
|||
|
||||
detectFs = {
|
||||
local = "test -e ${localBackendPath}/s3ql_metadata";
|
||||
s3c4 = pkgs.writeShellScript "detect-s3ql-filesystem" ''
|
||||
export AWS_ACCESS_KEY_ID="$(${pkgs.gnugrep}/bin/grep -m1 backend-login: '${authFile}' | cut -d' ' -f2-)"
|
||||
export AWS_SECRET_ACCESS_KEY="$(${pkgs.gnugrep}/bin/grep -m1 backend-password: '${authFile}' | cut -d' ' -f2-)"
|
||||
${pkgs.s5cmd}/bin/s5cmd --endpoint-url https://${s3Endpoint}/ ls 's3://${s3Bucket}/s3ql_params' >/dev/null
|
||||
'';
|
||||
}.${fsType} or null;
|
||||
in pkgs.writeShellScript "create-s3ql-filesystem" (lib.optionalString (detectFs != null) ''
|
||||
if ! ${detectFs}; then
|
||||
|
|
|
@ -22,6 +22,10 @@ with lib;
|
|||
authFile = mkOption {
|
||||
type = types.path;
|
||||
};
|
||||
locksmithSecret = mkOption {
|
||||
type = with types; nullOr str;
|
||||
default = null;
|
||||
};
|
||||
cacheDir = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/cache/remote-storage/${name}";
|
||||
|
|
|
@ -1,21 +1,32 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
perSystem = { config, ... }: {
|
||||
catalog.depot = {
|
||||
checks = lib.mapAttrs (name: check: {
|
||||
description = "NixOS Test: ${name}";
|
||||
actions = {
|
||||
build = {
|
||||
description = "Build this check.";
|
||||
command = "nix build -L --no-link '${builtins.unsafeDiscardStringContext check.drvPath}^*'";
|
||||
};
|
||||
runInteractive = {
|
||||
description = "Run interactive driver.";
|
||||
command = lib.getExe check.driverInteractive;
|
||||
};
|
||||
perSystem = { config, pkgs, ... }: {
|
||||
catalog = lib.mkMerge (lib.mapAttrsToList (name': check: let
|
||||
simulacrum = lib.hasPrefix "simulacrum-" name';
|
||||
name = lib.removePrefix "simulacrum-" name';
|
||||
baseAttrPath = if simulacrum then
|
||||
[ "cluster" "simulacrum" ]
|
||||
else
|
||||
[ "depot" "checks" ];
|
||||
in lib.setAttrByPath (baseAttrPath ++ [ name ]) {
|
||||
description = if simulacrum then
|
||||
"Simulacrum Test: ${name}"
|
||||
else
|
||||
"NixOS Test: ${name}";
|
||||
actions = {
|
||||
build = {
|
||||
description = "Build this check.";
|
||||
command = "nix build -L --no-link '${builtins.unsafeDiscardStringContext check.drvPath}^*'";
|
||||
};
|
||||
}) config.checks;
|
||||
};
|
||||
runInteractive = {
|
||||
description = "Run interactive driver.";
|
||||
command = if simulacrum then
|
||||
"${pkgs.bubblewrap}/bin/bwrap --unshare-all --bind / / --dev-bind /dev /dev ${lib.getExe check.driverInteractive}"
|
||||
else
|
||||
lib.getExe check.driverInteractive;
|
||||
};
|
||||
};
|
||||
}) config.checks);
|
||||
};
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ let
|
|||
in
|
||||
|
||||
{
|
||||
debug = lib.warn "debug mode is enabled" true;
|
||||
perSystem = { filters, pkgs, self', system, ... }: {
|
||||
checks = lib.mkIf (system == "x86_64-linux") {
|
||||
ascensions = pkgs.callPackage ./ascensions.nix {
|
||||
|
@ -15,12 +16,6 @@ in
|
|||
inherit (config) cluster;
|
||||
};
|
||||
|
||||
garage = pkgs.callPackage ./garage.nix {
|
||||
inherit (self'.packages) garage consul;
|
||||
inherit (self) nixosModules;
|
||||
inherit (config) cluster;
|
||||
};
|
||||
|
||||
ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
|
||||
inherit (self) nixosModules;
|
||||
previous = timeMachine.preUnstable;
|
||||
|
|
|
@ -1,155 +0,0 @@
|
|||
{ testers, nixosModules, cluster, garage, consul }:
|
||||
|
||||
testers.runNixOSTest {
|
||||
name = "garage";
|
||||
|
||||
imports = [
|
||||
./modules/consul.nix
|
||||
];
|
||||
|
||||
extraBaseModules.services.consul.package = consul;
|
||||
|
||||
nodes = let
|
||||
common = { config, lib, ... }: let
|
||||
inherit (config.networking) hostName primaryIPAddress;
|
||||
in {
|
||||
imports = lib.flatten [
|
||||
./modules/nixos/age-dummy-secrets
|
||||
./modules/nixos/age-dummy-secrets/options.nix
|
||||
nixosModules.ascensions
|
||||
nixosModules.systemd-extras
|
||||
nixosModules.consul-distributed-services
|
||||
nixosModules.port-magic
|
||||
cluster.config.services.storage.nixos.garage
|
||||
cluster.config.services.storage.nixos.garageInternal
|
||||
cluster.config.services.consul.nixos.ready
|
||||
];
|
||||
options.services.locksmith.providers = lib.mkOption {
|
||||
type = lib.types.raw;
|
||||
};
|
||||
config = {
|
||||
links.consulAgent = {
|
||||
protocol = "http";
|
||||
hostname = "consul";
|
||||
port = 8500;
|
||||
};
|
||||
_module.args = {
|
||||
depot.packages = { inherit garage; };
|
||||
cluster.config = {
|
||||
hostLinks.${hostName} = {
|
||||
garageRpc.tuple = "${primaryIPAddress}:3901";
|
||||
garageS3.tuple = "${primaryIPAddress}:8080";
|
||||
garageWeb.tuple = "${primaryIPAddress}:8081";
|
||||
};
|
||||
links.garageWeb.hostname = "web.garage.example.com";
|
||||
vars.meshNet.cidr = "192.168.0.0/16";
|
||||
};
|
||||
};
|
||||
environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
networking.firewall.allowedTCPPorts = [ 3901 8080 ];
|
||||
services.garage = {
|
||||
layout.initial = lib.mkOverride 51 {
|
||||
garage1 = { zone = "dc1"; capacity = 1000; };
|
||||
garage2 = { zone = "dc1"; capacity = 1000; };
|
||||
garage3 = { zone = "dc1"; capacity = 1000; };
|
||||
};
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
|
||||
specialisation.modifiedLayout = {
|
||||
inheritParentConfig = true;
|
||||
configuration = {
|
||||
services.garage = {
|
||||
layout.initial = lib.mkForce {
|
||||
garage1 = { zone = "dc1"; capacity = 2000; };
|
||||
garage2 = { zone = "dc1"; capacity = 1000; };
|
||||
garage3 = { zone = "dc1"; capacity = 1000; };
|
||||
};
|
||||
keys.testKey.allow.createBucket = true;
|
||||
buckets = {
|
||||
bucket1 = {
|
||||
allow.testKey = [ "read" "write" ];
|
||||
quotas = {
|
||||
maxObjects = 300;
|
||||
maxSize = 400 * 1024 * 1024;
|
||||
};
|
||||
};
|
||||
bucket2 = {
|
||||
allow.testKey = [ "read" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
system.ascensions.garage-layout.incantations = lib.mkForce (i: [
|
||||
(i.runGarage ''
|
||||
garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
|
||||
garage layout apply --version 2
|
||||
'')
|
||||
]);
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in {
|
||||
garage1.imports = [ common ];
|
||||
garage2.imports = [ common ];
|
||||
garage3.imports = [ common ];
|
||||
};
|
||||
|
||||
testScript = { nodes, ... }: /*python*/ ''
|
||||
nodes = [garage1, garage2, garage3]
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap new cluster"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
with subtest("should apply new layout with ascension"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
|
||||
|
||||
for node in nodes:
|
||||
node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should apply new layout from scratch"):
|
||||
for node in nodes:
|
||||
node.systemctl("stop garage.service")
|
||||
node.succeed("rm -rf /var/lib/garage-metadata")
|
||||
|
||||
for node in nodes:
|
||||
node.systemctl("start garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_for_unit("garage.service")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
|
||||
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds("garage layout show | grep -w 2000")
|
||||
assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
|
||||
assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
|
||||
|
||||
with subtest("should create specified buckets and keys"):
|
||||
for node in nodes:
|
||||
node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
|
||||
garage1.succeed("garage key list | grep testKey")
|
||||
garage1.succeed("garage bucket list | grep bucket1")
|
||||
garage1.succeed("garage bucket list | grep bucket2")
|
||||
|
||||
with subtest("should delete unspecified buckets and keys"):
|
||||
garage1.succeed("garage bucket create unwantedbucket")
|
||||
garage1.succeed("garage key new --name unwantedkey")
|
||||
garage1.succeed("systemctl restart garage-apply.service")
|
||||
|
||||
garage1.fail("garage key list | grep unwantedkey")
|
||||
garage1.fail("garage bucket list | grep unwantedbucket")
|
||||
'';
|
||||
}
|
12
packages/checks/modules/nixos/external-storage.nix
Normal file
12
packages/checks/modules/nixos/external-storage.nix
Normal file
|
@ -0,0 +1,12 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
systemd.tmpfiles.settings."00-testing-external-storage-underlays" = lib.mapAttrs' (name: cfg: {
|
||||
name = cfg.mountpoint;
|
||||
value.d = {
|
||||
user = toString cfg.uid;
|
||||
group = toString cfg.gid;
|
||||
mode = "0700";
|
||||
};
|
||||
}) config.services.external-storage.underlays;
|
||||
}
|
7
packages/checks/snakeoil/ssh/snakeoil-key
Normal file
7
packages/checks/snakeoil/ssh/snakeoil-key
Normal file
|
@ -0,0 +1,7 @@
|
|||
-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
|
||||
QyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgwAAAJAS78fWEu/H
|
||||
1gAAAAtzc2gtZWQyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgw
|
||||
AAAEAUtGOZZIZdzGP6g85JuXBjDtciNQ9bLHNxSN5Gbwvb2Q7HTdf4u1bRo3x6N03ggmAM
|
||||
+tNmBXB7tZteGEG+pXCDAAAACW1heEBUSVRBTgECAwQ=
|
||||
-----END OPENSSH PRIVATE KEY-----
|
1
packages/checks/snakeoil/ssh/snakeoil-key.pub
Normal file
1
packages/checks/snakeoil/ssh/snakeoil-key.pub
Normal file
|
@ -0,0 +1 @@
|
|||
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA7HTdf4u1bRo3x6N03ggmAM+tNmBXB7tZteGEG+pXCD
|
Loading…
Reference in a new issue