Compare commits
52 commits
master
...
pr-frangic
Author | SHA1 | Date | |
---|---|---|---|
00f233e8a5 | |||
e2fe73039c | |||
e4f09218d5 | |||
ebc9b88c8d | |||
ea3414c427 | |||
6eee030b7a | |||
75cebf4ed6 | |||
bc3cd82731 | |||
9cdf964c6c | |||
bb22fe0176 | |||
d1f2bc1227 | |||
a48ee00f3d | |||
9ed3655ccf | |||
945698a3ea | |||
f75c7b8522 | |||
b258bab23e | |||
e2296eace7 | |||
304ae6e53c | |||
f322208f66 | |||
7c4615ecfb | |||
ec38e10fa9 | |||
5d9ff62afe | |||
6d78b69601 | |||
7129d44078 | |||
76d205d114 | |||
c8c9a6fbce | |||
a1cad2efcd | |||
c7f4e59908 | |||
baed1ce871 | |||
014c1f9cd2 | |||
34704c8f08 | |||
ccc2a47880 | |||
05cd729e90 | |||
8d0a2f00cc | |||
ff26e1ebc1 | |||
b848084dd8 | |||
fe8ddd4094 | |||
030b680b33 | |||
b453b0bb21 | |||
b6e0390555 | |||
bbe3373c2e | |||
0ed4870b65 | |||
8ec13f5c87 | |||
5d52f72940 | |||
1af67b80ed | |||
a810717843 | |||
bd39fc5d07 | |||
25c001c182 | |||
d944dee3bc | |||
aac5163a8b | |||
01c74f62cf | |||
0110a4a0c3 |
75 changed files with 1101 additions and 1075 deletions
|
@ -46,6 +46,7 @@ in
|
|||
};
|
||||
}) // (if secretConfig.shared then let
|
||||
secretFile = "${svcName}-${secretName}.age";
|
||||
snakeoilFile = "${svcName}-${secretName}-snakeoil.txt";
|
||||
in {
|
||||
editSecret = {
|
||||
description = "Edit this secret";
|
||||
|
@ -54,7 +55,14 @@ in
|
|||
agenix -e '${secretFile}'
|
||||
'';
|
||||
};
|
||||
} else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
|
||||
editSnakeoil = {
|
||||
description = "Edit this secret's snakeoil";
|
||||
command = ''
|
||||
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
|
||||
'';
|
||||
};
|
||||
} else lib.mkMerge [
|
||||
(lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
|
||||
secretFile = "${svcName}-${secretName}-${node}.age";
|
||||
in {
|
||||
description = "Edit this secret for '${node}'";
|
||||
|
@ -62,7 +70,16 @@ in
|
|||
${setupCommands secretFile [ node ]}
|
||||
agenix -e '${secretFile}'
|
||||
'';
|
||||
})));
|
||||
})))
|
||||
(lib.mapAttrs' (name: lib.nameValuePair "editSnakeoilInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
|
||||
snakeoilFile = "${svcName}-${secretName}-${node}-snakeoil.txt";
|
||||
in {
|
||||
description = "Edit this secret's snakeoil for '${node}'";
|
||||
command = ''
|
||||
$EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
|
||||
'';
|
||||
})))
|
||||
]);
|
||||
};
|
||||
}) svcConfig.secrets))
|
||||
lib.concatLists
|
||||
|
|
|
@ -3,14 +3,12 @@
|
|||
{
|
||||
hostLinks = lib.pipe config.services [
|
||||
(lib.filterAttrs (_: svc: svc.meshLinks != {}))
|
||||
(lib.mapAttrsToList (svcName: svc:
|
||||
lib.mapAttrsToList (groupName: links:
|
||||
lib.genAttrs svc.nodes.${groupName} (hostName: lib.mapAttrs (_: cfg: { ... }: {
|
||||
(lib.mapAttrsToList (svcName: svc: lib.mapAttrsToList (name: cfg: lib.genAttrs svc.nodes.${name} (hostName: {
|
||||
${cfg.name} = { ... }: {
|
||||
imports = [ cfg.link ];
|
||||
ipv4 = config.vars.mesh.${hostName}.meshIp;
|
||||
}) links)
|
||||
) svc.meshLinks
|
||||
))
|
||||
};
|
||||
})) svc.meshLinks))
|
||||
(map lib.mkMerge)
|
||||
lib.mkMerge
|
||||
];
|
||||
|
|
|
@ -38,8 +38,12 @@ in
|
|||
};
|
||||
meshLinks = mkOption {
|
||||
description = "Create host links on the mesh network.";
|
||||
type = types.attrsOf (types.attrsOf (types.submodule {
|
||||
type = types.attrsOf (types.submodule ({ name, ... }: {
|
||||
options = {
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
default = "${serviceName}-${name}";
|
||||
};
|
||||
link = mkOption {
|
||||
type = types.deferredModule;
|
||||
default = {};
|
||||
|
|
|
@ -74,7 +74,7 @@ in
|
|||
serviceConfig = {
|
||||
Restart = "on-failure";
|
||||
RestartMaxDelaySec = 30;
|
||||
RestartSteps = 5;
|
||||
RestartStesp = 5;
|
||||
RestartMode = "direct";
|
||||
};
|
||||
};
|
||||
|
|
|
@ -16,7 +16,10 @@
|
|||
./nar-serve.nix
|
||||
];
|
||||
};
|
||||
meshLinks.server.attic.link.protocol = "http";
|
||||
meshLinks.server = {
|
||||
name = "attic";
|
||||
link.protocol = "http";
|
||||
};
|
||||
secrets = let
|
||||
inherit (config.services.attic) nodes;
|
||||
in {
|
||||
|
@ -49,12 +52,10 @@
|
|||
cache.target = serverAddrs;
|
||||
};
|
||||
|
||||
ways = config.lib.forService "attic" {
|
||||
cache-api = {
|
||||
ways.cache-api = {
|
||||
consulService = "atticd";
|
||||
extras.extraConfig = ''
|
||||
client_max_body_size 4G;
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -9,13 +9,17 @@ let
|
|||
in
|
||||
|
||||
{
|
||||
imports = [
|
||||
depot.inputs.attic.nixosModules.atticd
|
||||
];
|
||||
|
||||
services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];
|
||||
|
||||
services.atticd = {
|
||||
enable = true;
|
||||
package = depot.inputs.attic.packages.attic-server;
|
||||
|
||||
environmentFile = secrets.serverToken.path;
|
||||
credentialsFile = secrets.serverToken.path;
|
||||
mode = if isMonolith then "monolithic" else "api-server";
|
||||
|
||||
settings = {
|
||||
|
@ -65,7 +69,6 @@ in
|
|||
serviceConfig = {
|
||||
DynamicUser = lib.mkForce false;
|
||||
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
|
||||
SystemCallFilter = lib.mkAfter [ "@resources" ];
|
||||
};
|
||||
environment = {
|
||||
AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
|
||||
|
@ -77,7 +80,6 @@ in
|
|||
mode = if isMonolith then "manual" else "direct";
|
||||
definition = {
|
||||
name = "atticd";
|
||||
id = "atticd-${config.services.atticd.mode}";
|
||||
address = link.ipv4;
|
||||
inherit (link) port;
|
||||
checks = [
|
||||
|
|
|
@ -10,6 +10,13 @@ let
|
|||
in
|
||||
|
||||
{
|
||||
system.extraIncantations = {
|
||||
runConsul = i: script: i.execShellWith [ config.services.consul.package ] ''
|
||||
export CONSUL_HTTP_ADDR='${config.links.consulAgent.tuple}'
|
||||
${script}
|
||||
'';
|
||||
};
|
||||
|
||||
links.consulAgent.protocol = "http";
|
||||
|
||||
services.consul = {
|
||||
|
|
|
@ -1,24 +1,19 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should form cluster"):
|
||||
nodes = [ n for n in machines if n != nowhere ]
|
||||
for machine in nodes:
|
||||
for machine in machines:
|
||||
machine.succeed("systemctl start consul-ready.service")
|
||||
for machine in nodes:
|
||||
for machine in machines:
|
||||
consulConfig = json.loads(machine.succeed("cat /etc/consul.json"))
|
||||
addr = consulConfig["addresses"]["http"]
|
||||
port = consulConfig["ports"]["http"]
|
||||
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
|
||||
memberList = machine.succeed(f"{setEnv} consul members --status=alive")
|
||||
for machine2 in nodes:
|
||||
for machine2 in machines:
|
||||
assert machine2.name in memberList
|
||||
'';
|
||||
}
|
||||
|
|
16
cluster/services/dns/acme-dns-db-credentials.age
Normal file
16
cluster/services/dns/acme-dns-db-credentials.age
Normal file
|
@ -0,0 +1,16 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 NO562A YndVtONpmfFXYB1ASnPHsfczl1UbgZ2vccIrX2pEgx0
|
||||
VzH2UD583L6wBLMCo6faIGyHR4+zXXOUTgQduEiFOxI
|
||||
-> ssh-ed25519 5/zT0w +67r5S6PSFEgnrTu3eZpOd3eemZUdDOE+kjUw6GDgUM
|
||||
jPzlW7hePFgsABUjryePu5yergQ2Qjczmmoxuo6CK+U
|
||||
-> ssh-ed25519 TCgorQ DGJPjJYpeibxM+8OwofUCdttIT2OdNbvQ66wpWQM8XU
|
||||
JCNQ3bT21j2ZsxbzA6FieKIui6lsvk1p0nvNOT7YtFo
|
||||
-> ssh-ed25519 d3WGuA hIl5yluwf1f0DP5ZW1MalGPCj4XFYOu2sofwJSQZ6RE
|
||||
BSHoe4cdRJlPrkc+taUIaIIUknexlGttzz2d9I3jtmk
|
||||
-> ssh-ed25519 YIaSKQ EbqXS/XFQHSXCbzDJmg4gGUxP9TX3+vOxWtNQDJ8ih4
|
||||
hNaWzoFG2iVef4Gm30LilGXYNsVkhmVt9dOvBo02mbM
|
||||
-> V]i@xRtJ-grease
|
||||
NEPxMUZa76GclWOasWptt6QS7frMclp9o+kD4KCLJB7ucFOYK7xxWfAEMkjtadfP
|
||||
m0bbgbw7Jcs9/lA8VNAG2D5jTBayGgpkBQZ4
|
||||
--- ViqZD8mJEKIMCZ5Q+wRQWR2FX/LMEfUwoumUtHlYabQ
|
||||
KAÉû¹ÝgZü<šë*DfV6·=äG»+eœ`ºpª±ï÷6°<1E>º[Û‘Û û¸¢ºÐý-H1<1B>»Ã›Íí[fV.¾¢HÁ"OhÐñŒ½j•ùö8ïßß$‰;Û‘&5<>äxw§/mŒë<C592>Ö‘ß^7î‘f5ÔµyÏŽÓûC‚´6”¹U•æýi-R=/_R<5F><52>„·==æà½1˜'Ò qÞ·ŒvÜcwø
|
|
@ -35,13 +35,10 @@ in
|
|||
];
|
||||
before = [ "acme-securedns.${domain}.service" ];
|
||||
wants = [ "acme-finished-securedns.${domain}.target" ];
|
||||
serviceConfig = {
|
||||
LoadCredential = [
|
||||
serviceConfig.LoadCredential = [
|
||||
"dot-cert.pem:${dot.directory}/fullchain.pem"
|
||||
"dot-key.pem:${dot.directory}/key.pem"
|
||||
];
|
||||
ExecReload = lib.mkForce [];
|
||||
};
|
||||
};
|
||||
|
||||
security.acme.certs."securedns.${domain}" = {
|
||||
|
|
|
@ -56,11 +56,6 @@ in
|
|||
coredns = ./coredns.nix;
|
||||
client = ./client.nix;
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "consul" "acme-client" "patroni" ];
|
||||
settings = ./test.nix;
|
||||
};
|
||||
};
|
||||
|
||||
patroni = {
|
||||
|
|
|
@ -1,35 +0,0 @@
|
|||
{ cluster, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster._module.specialArgs.depot.lib.meta) domain;
|
||||
in
|
||||
{
|
||||
nodes.nowhere = { pkgs, ... }: {
|
||||
passthru = cluster;
|
||||
environment.systemPackages = [
|
||||
pkgs.knot-dns
|
||||
pkgs.openssl
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
nodeNames = json.loads('${builtins.toJSON cluster.config.services.dns.nodes.authoritative}')
|
||||
dotNames = json.loads('${builtins.toJSON cluster.config.services.dns.nodes.coredns}')
|
||||
nodes = [ n for n in machines if n.name in nodeNames ]
|
||||
dotServers = [ n for n in machines if n.name in dotNames ]
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should allow external name resolution for own domain"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("coredns.service")
|
||||
nowhere.wait_until_succeeds("[[ $(kdig +short securedns.${domain} | wc -l) -ne 0 ]]", timeout=60)
|
||||
nowhere.fail("[[ $(kdig +short example.com | wc -l) -ne 0 ]]")
|
||||
|
||||
with subtest("should have valid certificate on DoT endpoint"):
|
||||
for node in dotServers:
|
||||
node.wait_for_unit("acme-finished-securedns.${domain}.target")
|
||||
nowhere.wait_until_succeeds("openssl </dev/null s_client -connect securedns.${domain}:853 -verify_return_error -strict -verify_hostname securedns.${domain}", timeout=60)
|
||||
'';
|
||||
}
|
|
@ -4,7 +4,10 @@
|
|||
services.forge = {
|
||||
nodes.server = [ "VEGAS" ];
|
||||
nixos.server = ./server.nix;
|
||||
meshLinks.server.forge.link.protocol = "http";
|
||||
meshLinks.server = {
|
||||
name = "forge";
|
||||
link.protocol = "http";
|
||||
};
|
||||
secrets = with config.services.forge.nodes; {
|
||||
oidcSecret = {
|
||||
nodes = server;
|
||||
|
@ -13,10 +16,10 @@
|
|||
};
|
||||
};
|
||||
|
||||
ways = let
|
||||
ways.forge = let
|
||||
host = builtins.head config.services.forge.nodes.server;
|
||||
in config.lib.forService "forge" {
|
||||
forge.target = config.hostLinks.${host}.forge.url;
|
||||
target = config.hostLinks.${host}.forge.url;
|
||||
};
|
||||
|
||||
patroni = config.lib.forService "forge" {
|
||||
|
|
27
cluster/services/frangiclave/default.nix
Normal file
27
cluster/services/frangiclave/default.nix
Normal file
|
@ -0,0 +1,27 @@
|
|||
{ config, ... }:
|
||||
|
||||
{
|
||||
services.frangiclave = {
|
||||
nodes = {
|
||||
server = [ "VEGAS" "grail" "prophet" ];
|
||||
cluster = config.services.frangiclave.nodes.server;
|
||||
agent = []; # all nodes, for vault-agent, secret templates, etc.
|
||||
};
|
||||
meshLinks = {
|
||||
server.link.protocol = "http";
|
||||
cluster.link.protocol = "http";
|
||||
};
|
||||
nixos = {
|
||||
server = [
|
||||
./server.nix
|
||||
];
|
||||
cluster = [];
|
||||
agent = [];
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "wireguard" "consul" ];
|
||||
settings = ./test.nix;
|
||||
};
|
||||
};
|
||||
}
|
34
cluster/services/frangiclave/server.nix
Normal file
34
cluster/services/frangiclave/server.nix
Normal file
|
@ -0,0 +1,34 @@
|
|||
{ cluster, config, depot, lib, ... }:
|
||||
|
||||
let
|
||||
apiLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server;
|
||||
clusterLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-cluster;
|
||||
in
|
||||
|
||||
{
|
||||
services.vault = {
|
||||
enable = true;
|
||||
package = depot.packages.openbao;
|
||||
address = apiLink.tuple;
|
||||
extraConfig = /*hcl*/ ''
|
||||
api_addr = "${apiLink.url}"
|
||||
cluster_addr = "${clusterLink.url}"
|
||||
'';
|
||||
|
||||
storageBackend = "raft";
|
||||
storageConfig = /*hcl*/ ''
|
||||
node_id = "x${builtins.hashString "sha256" "frangiclave-node-${config.networking.hostName}"}"
|
||||
${
|
||||
lib.pipe (cluster.config.services.frangiclave.otherNodes.server config.networking.hostName) [
|
||||
(map (node: cluster.config.hostLinks.${node}.frangiclave-server))
|
||||
(map (link: /*hcl*/ ''
|
||||
retry_join {
|
||||
leader_api_addr = "${link.url}"
|
||||
}
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]
|
||||
}
|
||||
'';
|
||||
};
|
||||
}
|
12
cluster/services/frangiclave/test.nix
Normal file
12
cluster/services/frangiclave/test.nix
Normal file
|
@ -0,0 +1,12 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
interactive.defaults = { cluster, config, ... }: {
|
||||
config = lib.mkIf config.services.vault.enable {
|
||||
environment.variables.VAULT_ADDR = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server.url;
|
||||
environment.systemPackages = [ config.services.vault.package ];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = "assert False";
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
{ cluster, config, lib, pkgs, utils, ... }:
|
||||
{ cluster, config, pkgs, utils, ... }:
|
||||
|
||||
let
|
||||
frontendLink = cluster.config.links.idm;
|
||||
|
@ -39,8 +39,9 @@ in
|
|||
security = {
|
||||
pam.services.sudo = { config, ... }: {
|
||||
rules.auth.rssh = {
|
||||
enable = lib.mkForce true;
|
||||
order = config.rules.auth.unix.order - 10;
|
||||
control = "sufficient";
|
||||
modulePath = "${pkgs.pam_rssh}/lib/libpam_rssh.so";
|
||||
settings = {
|
||||
authorized_keys_command = "/etc/ssh/authorized_keys_command_kanidm";
|
||||
authorized_keys_command_user = "nobody";
|
||||
|
|
|
@ -33,9 +33,6 @@ in
|
|||
ldapbindaddress = "${ldapLink.ipv4}:${ldapLink.portStr}";
|
||||
origin = frontendLink.url;
|
||||
inherit domain;
|
||||
online_backup = {
|
||||
versions = 7;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
@ -3,7 +3,6 @@
|
|||
{
|
||||
imports = [
|
||||
./options.nix
|
||||
./simulacrum/test-data.nix
|
||||
];
|
||||
|
||||
services.incandescence = {
|
||||
|
@ -16,10 +15,6 @@
|
|||
./provider-options.nix
|
||||
];
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "consul" "locksmith" ];
|
||||
settings = ./simulacrum/test.nix;
|
||||
};
|
||||
simulacrum.deps = [ "consul" ];
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
{ cluster, config, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) concatStringsSep escapeShellArg flatten filter filterAttrs length mapAttrs mapAttrs' mapAttrsToList mkIf mkMerge optionalString pipe stringToCharacters;
|
||||
inherit (lib) concatStringsSep escapeShellArg flatten filter filterAttrs length mapAttrs mapAttrs' mapAttrsToList mkIf mkMerge pipe stringToCharacters;
|
||||
|
||||
cfg = config.services.incandescence;
|
||||
clusterCfg = cluster.config.incandescence;
|
||||
|
@ -39,9 +39,6 @@ in
|
|||
fi
|
||||
''))
|
||||
(concatStringsSep "\n")
|
||||
(script: if script == "" then ''
|
||||
echo "Nothing to create"
|
||||
'' else script)
|
||||
];
|
||||
};
|
||||
"ignite-${provider}-${formula}-change" = mkIf (formulaConfig.change != null) {
|
||||
|
@ -61,16 +58,14 @@ in
|
|||
) || echo "Change failed: ${object}"
|
||||
''))
|
||||
(concatStringsSep "\n")
|
||||
(script: if script == "" then ''
|
||||
echo "Nothing to change"
|
||||
'' else script)
|
||||
];
|
||||
};
|
||||
"ignite-${provider}-${formula}-destroy" = {
|
||||
description = "Ignite Destruction: ${provider} - ${formula}";
|
||||
wantedBy = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
|
||||
before = [ "incandescence-${provider}.target" ] ++ map (dep: "ignite-${provider}-${dep}-destroy.service") formulaConfig.deps;
|
||||
inherit (providerConfig) wants after;
|
||||
wants = providerConfig.wants ++ [ "ignite-${provider}-${formula}-change.service" ];
|
||||
after = providerConfig.after ++ [ "ignite-${provider}-${formula}-change.service" ];
|
||||
serviceConfig.Type = "oneshot";
|
||||
distributed.enable = true;
|
||||
path = [ config.services.consul.package ] ++ providerConfig.packages;
|
||||
|
@ -82,15 +77,13 @@ in
|
|||
(builtins.add 2)
|
||||
toString
|
||||
];
|
||||
needsFilter = clusterCfg.providers.${provider}.objects.${formula} != [];
|
||||
keyFilter = pipe clusterCfg.providers.${provider}.objects.${formula} [
|
||||
(map (x: escapeShellArg "^${x}$"))
|
||||
(concatStringsSep " \\\n -e ")
|
||||
];
|
||||
destroyAfterDays = toString formulaConfig.destroyAfterDays;
|
||||
grep = optionalString needsFilter "grep -v -e ${keyFilter} |";
|
||||
in ''
|
||||
consul kv get --keys ${kvRoot}/ | cut -d/ -f${fieldNum} | ${grep} while read object; do
|
||||
consul kv get --keys ${kvRoot}/ | cut -d/ -f${fieldNum} | grep -v -e ${keyFilter} | while read object; do
|
||||
if consul kv get ${kvRoot}/$object/alive >/dev/null; then
|
||||
destroyOn="$(consul kv get ${kvRoot}/$object/destroyOn || true)"
|
||||
if [[ -z "$destroyOn" && "${destroyAfterDays}" -ne 0 ]]; then
|
||||
|
|
|
@ -1,8 +0,0 @@
|
|||
{ config, lib, ... }:
|
||||
{
|
||||
incandescence = lib.mkIf config.simulacrum {
|
||||
providers = config.lib.forService "incandescence" {
|
||||
test.objects.example = [ "example1" "example2" ];
|
||||
};
|
||||
};
|
||||
}
|
|
@ -1,47 +0,0 @@
|
|||
{ cluster, lib, ... }:
|
||||
|
||||
let
|
||||
providers = lib.take 2 cluster.config.services.incandescence.nodes.provider;
|
||||
in
|
||||
|
||||
{
|
||||
nodes = lib.genAttrs providers (lib.const {
|
||||
services.incandescence.providers.test = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
partOf = [ ];
|
||||
formulae.example = {
|
||||
create = x: "consul kv put testData/${x} ${x}";
|
||||
destroy = "consul kv delete testData/$OBJECT";
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
nodeNames = json.loads('${builtins.toJSON providers}')
|
||||
nodes = [ n for n in machines if n.name in nodeNames ]
|
||||
|
||||
start_all()
|
||||
|
||||
consulConfig = json.loads(nodes[0].succeed("cat /etc/consul.json"))
|
||||
addr = consulConfig["addresses"]["http"]
|
||||
port = consulConfig["ports"]["http"]
|
||||
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
|
||||
|
||||
with subtest("should create objects"):
|
||||
for node in nodes:
|
||||
node.wait_for_unit("incandescence-test.target")
|
||||
nodes[0].succeed(f"[[ $({setEnv} consul kv get testData/example1) == example1 ]]")
|
||||
nodes[0].succeed(f"[[ $({setEnv} consul kv get testData/example2) == example2 ]]")
|
||||
|
||||
with subtest("should destroy objects"):
|
||||
nodes[0].succeed(f"{setEnv} consul kv put testData/example3 example3")
|
||||
nodes[0].succeed(f"{setEnv} consul kv put services/incandescence/providers/test/formulae/example/example3/alive true")
|
||||
nodes[1].succeed(f"{setEnv} consul kv get testData/example3")
|
||||
for node in nodes:
|
||||
node.systemctl("isolate default")
|
||||
for node in nodes:
|
||||
node.wait_for_unit("incandescence-test.target")
|
||||
nodes[0].fail(f"{setEnv} consul kv get testData/example3")
|
||||
'';
|
||||
}
|
|
@ -29,7 +29,10 @@
|
|||
io-tweaks = [ "VEGAS" ];
|
||||
remote-api = [ "VEGAS" ];
|
||||
};
|
||||
meshLinks.gateway.ipfsGateway.link.protocol = "http";
|
||||
meshLinks.gateway = {
|
||||
name = "ipfsGateway";
|
||||
link.protocol = "http";
|
||||
};
|
||||
nixos = {
|
||||
node = [
|
||||
./node.nix
|
||||
|
|
|
@ -9,7 +9,7 @@ in
|
|||
environment = {
|
||||
OTEL_TRACES_EXPORTER = "otlp";
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "${cluster.config.ways.ingest-traces-otlp.url}:443";
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
|
||||
OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
|
||||
OTEL_TRACES_SAMPLER_ARG = "0.50";
|
||||
};
|
||||
|
|
|
@ -26,7 +26,7 @@ in {
|
|||
name = "logging";
|
||||
positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
|
||||
clients = singleton {
|
||||
url = "${cluster.config.ways.ingest-logs.url}/loki/api/v1/push";
|
||||
url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
|
||||
};
|
||||
scrape_configs = singleton {
|
||||
job_name = "journal";
|
||||
|
|
|
@ -18,6 +18,26 @@ in
|
|||
protocol = "http";
|
||||
ipv4 = meshIpFor "server";
|
||||
};
|
||||
tempo = {
|
||||
protocol = "http";
|
||||
ipv4 = meshIpFor "server";
|
||||
};
|
||||
tempo-grpc = {
|
||||
protocol = "http";
|
||||
ipv4 = "127.0.0.1";
|
||||
};
|
||||
tempo-otlp-http = {
|
||||
protocol = "http";
|
||||
ipv4 = meshIpFor "server";
|
||||
};
|
||||
tempo-otlp-grpc = {
|
||||
protocol = "http";
|
||||
ipv4 = meshIpFor "server";
|
||||
};
|
||||
tempo-zipkin-http = {
|
||||
protocol = "http";
|
||||
ipv4 = meshIpFor "server";
|
||||
};
|
||||
};
|
||||
hostLinks = lib.genAttrs config.services.monitoring.nodes.grafana (name: {
|
||||
grafana = {
|
||||
|
@ -31,7 +51,6 @@ in
|
|||
blackbox = [ "checkmate" "grail" "prophet" ];
|
||||
grafana = [ "VEGAS" "prophet" ];
|
||||
logging = [ "VEGAS" "grail" ];
|
||||
tracing = [ "VEGAS" "grail" ];
|
||||
server = [ "VEGAS" ];
|
||||
};
|
||||
nixos = {
|
||||
|
@ -42,19 +61,14 @@ in
|
|||
./provisioning/dashboards.nix
|
||||
];
|
||||
logging = ./logging.nix;
|
||||
tracing = ./tracing.nix;
|
||||
server = [
|
||||
./server.nix
|
||||
./tracing.nix
|
||||
];
|
||||
};
|
||||
meshLinks = {
|
||||
logging.loki.link.protocol = "http";
|
||||
tracing = {
|
||||
tempo.link.protocol = "http";
|
||||
tempo-otlp-http.link.protocol = "http";
|
||||
tempo-otlp-grpc.link.protocol = "grpc";
|
||||
tempo-zipkin-http.link.protocol = "http";
|
||||
};
|
||||
meshLinks.logging = {
|
||||
name = "loki";
|
||||
link.protocol = "http";
|
||||
};
|
||||
};
|
||||
|
||||
|
@ -68,51 +82,29 @@ in
|
|||
nodes = config.services.monitoring.nodes.logging;
|
||||
format = "envFile";
|
||||
};
|
||||
tempo-ingest.locksmith = {
|
||||
nodes = config.services.monitoring.nodes.tracing;
|
||||
format = "envFile";
|
||||
};
|
||||
tempo-query.locksmith = {
|
||||
nodes = config.services.monitoring.nodes.tracing;
|
||||
format = "envFile";
|
||||
};
|
||||
tempo = { };
|
||||
};
|
||||
buckets = {
|
||||
loki-chunks.allow = {
|
||||
loki-ingest = [ "read" "write" ];
|
||||
loki-query = [ "read" ];
|
||||
};
|
||||
tempo-chunks.allow = {
|
||||
tempo-ingest = [ "read" "write" ];
|
||||
tempo-query = [ "read" ];
|
||||
};
|
||||
tempo-chunks.allow.tempo = [ "read" "write" ];
|
||||
};
|
||||
};
|
||||
|
||||
ways = let
|
||||
query = consulService: {
|
||||
inherit consulService;
|
||||
internal = true;
|
||||
extras.extraConfig = ''
|
||||
proxy_read_timeout 3600s;
|
||||
'';
|
||||
ways = {
|
||||
monitoring = {
|
||||
consulService = "grafana";
|
||||
extras.locations."/".proxyWebsockets = true;
|
||||
};
|
||||
ingest = consulService: {
|
||||
inherit consulService;
|
||||
monitoring-logs = {
|
||||
internal = true;
|
||||
consulService = "loki";
|
||||
extras.extraConfig = ''
|
||||
client_max_body_size 4G;
|
||||
proxy_read_timeout 3600s;
|
||||
'';
|
||||
};
|
||||
in config.lib.forService "monitoring" {
|
||||
monitoring = {
|
||||
consulService = "grafana";
|
||||
extras.locations."/".proxyWebsockets = true;
|
||||
};
|
||||
monitoring-logs = query "loki";
|
||||
monitoring-traces = query "tempo";
|
||||
ingest-logs = ingest "loki";
|
||||
ingest-traces-otlp = ingest "tempo-ingest-otlp-grpc" // { grpc = true; };
|
||||
};
|
||||
}
|
||||
|
|
|
@ -73,16 +73,6 @@ in
|
|||
inherit (cluster.config.ways.monitoring-logs) url;
|
||||
type = "loki";
|
||||
}
|
||||
{
|
||||
name = "Tempo";
|
||||
uid = "P214B5B846CF3925F";
|
||||
inherit (cluster.config.ways.monitoring-traces) url;
|
||||
type = "tempo";
|
||||
jsonData = {
|
||||
serviceMap.datasourceUid = "PBFA97CFB590B2093";
|
||||
nodeGraph.enabled = true;
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
|
|
@ -19,23 +19,10 @@
|
|||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": 1210,
|
||||
"id": 16,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 7,
|
||||
"panels": [],
|
||||
"title": "Replication",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
|
@ -71,7 +58,7 @@
|
|||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
|
@ -79,7 +66,6 @@
|
|||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
|
@ -87,11 +73,9 @@
|
|||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "11.1.3",
|
||||
"pluginVersion": "9.5.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
|
@ -154,7 +138,7 @@
|
|||
"h": 8,
|
||||
"w": 4,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
"y": 0
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
|
@ -162,17 +146,14 @@
|
|||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [],
|
||||
"fields": "/^instance$/",
|
||||
"values": true
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "11.1.3",
|
||||
"pluginVersion": "9.5.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
|
@ -217,7 +198,7 @@
|
|||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 1
|
||||
"y": 0
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
|
@ -273,7 +254,6 @@
|
|||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
|
@ -287,7 +267,6 @@
|
|||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineStyle": {
|
||||
"fill": "solid"
|
||||
|
@ -325,7 +304,7 @@
|
|||
"h": 13,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
"y": 8
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
|
@ -369,7 +348,6 @@
|
|||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
|
@ -383,7 +361,6 @@
|
|||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
|
@ -421,7 +398,7 @@
|
|||
"h": 13,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
"y": 8
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
|
@ -451,326 +428,24 @@
|
|||
],
|
||||
"title": "Activity",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 22
|
||||
},
|
||||
"id": 8,
|
||||
"panels": [],
|
||||
"title": "Storage",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 15,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 23
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"displayLabels": [
|
||||
"percent"
|
||||
],
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true,
|
||||
"sortBy": "Value",
|
||||
"sortDesc": true,
|
||||
"values": [
|
||||
"percent",
|
||||
"value"
|
||||
]
|
||||
},
|
||||
"pieType": "donut",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "11.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "max by(datname) (sum by(datname, instance) (pg_database_size_bytes))",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": true,
|
||||
"legendFormat": "__auto",
|
||||
"range": false,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Database Size",
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "opacity",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 15,
|
||||
"w": 5,
|
||||
"x": 8,
|
||||
"y": 23
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "11.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "pg_wal_size_bytes",
|
||||
"format": "time_series",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": false,
|
||||
"legendFormat": "{{instance}}",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "WAL Size",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds",
|
||||
"seriesBy": "last"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 25,
|
||||
"gradientMode": "opacity",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineStyle": {
|
||||
"fill": "solid"
|
||||
},
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"decimals": 3,
|
||||
"fieldMinMax": false,
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 15,
|
||||
"w": 11,
|
||||
"x": 13,
|
||||
"y": 23
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": false
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "max(sum by(instance) (pg_database_size_bytes))",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": false,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Total DB Size",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "5m",
|
||||
"schemaVersion": 39,
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "PostgreSQL HA",
|
||||
"uid": "dc7545ef-3180-4a5e-a289-1e64571ebb87",
|
||||
"version": 2,
|
||||
"version": 7,
|
||||
"weekStart": ""
|
||||
},
|
||||
"folderId": 0,
|
||||
|
|
16
cluster/services/monitoring/secrets/tempo-secrets.age
Normal file
16
cluster/services/monitoring/secrets/tempo-secrets.age
Normal file
|
@ -0,0 +1,16 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 NO562A KhCGp7PAMGrEdzRxBrsW4tRk30JwpI+4lPzrRCUhSw4
|
||||
8s7WqA5c3zS1euN5R+jfFNBdvr8OQW8P4NFeqtNsIKo
|
||||
-> ssh-ed25519 5/zT0w 79hJQ2H76EZTW7YcQFCtKaS5Kbssx4Z8dPFjIVzRgFk
|
||||
A1fDJbUnyIRy+kWa3PhJNj/SdRPlcEy6FYsAfnuZ2AQ
|
||||
-> ssh-ed25519 d3WGuA aylkdL1KliM1NfrYDGlG8X6YjXvVUCU4sV90I+a840U
|
||||
6sXdqIPjtoNSylZRh1DCghHOwDo+fC7WB4QWQoWmG48
|
||||
-> //gd+2-grease baUWA$3 z-qs3W O/2.1W
|
||||
Sfq3+rkMJhpUTTmcos5TaaUtX2Ip9pciHAZLiWPix+C9N7ccac/1W5RNedMJCLsq
|
||||
MQ+xKzexf8+hgNVhKOksvbKBBROXqk1bUOKk8w3OgFPmmByzmCBUwkdkeu5DFTYR
|
||||
rg
|
||||
--- kUl1uIPRkM5y7C68kdN22pMKXP7gazyha4PE+ap0Jqw
|
||||
w>×Àè¥
|
||||
<15>CÈ,\‰ßœI¯ˆúHxG@^Çá“PåÃþÙÏlw6µŽ{þ’rb‘é5æ†T>Êñ
|
||||
ÚWܤX4Kp(ß?9ˆß^^oP3f </v3N$ê¤sÓbŽ¾>O™÷œ+òN0άïµàDtêŽ5Vº#è ¶³‘Uã îŸ#y|›@ŒGzSi»ô*·HùüŽ]
|
||||
ꎀ5›
|
|
@ -1,16 +1,14 @@
|
|||
{ cluster, config, pkgs, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster.config.links) prometheus-ingest;
|
||||
inherit (config.links) tempo-grpc;
|
||||
links = cluster.config.hostLinks.${config.networking.hostName};
|
||||
inherit (cluster.config) links;
|
||||
dataDir = "/srv/storage/private/tempo";
|
||||
tempoConfig = {
|
||||
server = {
|
||||
http_listen_address = links.tempo.ipv4;
|
||||
http_listen_port = links.tempo.port;
|
||||
grpc_listen_address = tempo-grpc.ipv4;
|
||||
grpc_listen_port = tempo-grpc.port;
|
||||
grpc_listen_address = links.tempo-grpc.ipv4;
|
||||
grpc_listen_port = links.tempo-grpc.port;
|
||||
};
|
||||
distributor.receivers = {
|
||||
otlp = {
|
||||
|
@ -21,7 +19,7 @@ let
|
|||
};
|
||||
zipkin.endpoint = links.tempo-zipkin-http.tuple;
|
||||
};
|
||||
querier.frontend_worker.frontend_address = tempo-grpc.tuple;
|
||||
querier.frontend_worker.frontend_address = links.tempo-grpc.tuple;
|
||||
ingester = {
|
||||
trace_idle_period = "30s";
|
||||
max_block_bytes = 1000000;
|
||||
|
@ -58,7 +56,7 @@ let
|
|||
path = "${dataDir}/generator/wal";
|
||||
remote_write = [
|
||||
{
|
||||
url = "${prometheus-ingest.url}/api/v1/write";
|
||||
url = "${links.prometheus-ingest.url}/api/v1/write";
|
||||
send_exemplars = true;
|
||||
}
|
||||
];
|
||||
|
@ -70,11 +68,7 @@ let
|
|||
];
|
||||
};
|
||||
in {
|
||||
links.tempo-grpc.protocol = "http";
|
||||
|
||||
services.locksmith.waitForSecrets.tempo = [
|
||||
"garage-tempo-ingest"
|
||||
];
|
||||
age.secrets.tempoSecrets.file = ./secrets/tempo-secrets.age;
|
||||
|
||||
users.users.tempo = {
|
||||
isSystemUser = true;
|
||||
|
@ -87,53 +81,24 @@ in {
|
|||
|
||||
systemd.services.tempo = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
distributed = {
|
||||
enable = true;
|
||||
registerServices = [
|
||||
"tempo"
|
||||
"tempo-ingest-otlp-grpc"
|
||||
];
|
||||
};
|
||||
serviceConfig = {
|
||||
User = "tempo";
|
||||
Group = "tempo";
|
||||
ExecStart = "${pkgs.tempo}/bin/tempo -config.file=${pkgs.writeText "tempo.yaml" (builtins.toJSON tempoConfig)}";
|
||||
PrivateTmp = true;
|
||||
EnvironmentFile = "/run/locksmith/garage-tempo-ingest";
|
||||
EnvironmentFile = config.age.secrets.tempoSecrets.path;
|
||||
};
|
||||
};
|
||||
|
||||
consul.services = {
|
||||
tempo = {
|
||||
mode = "manual";
|
||||
definition = {
|
||||
name = "tempo";
|
||||
address = links.tempo.ipv4;
|
||||
inherit (links.tempo) port;
|
||||
checks = [
|
||||
services.grafana.provision.datasources.settings.datasources = [
|
||||
{
|
||||
name = "Tempo";
|
||||
id = "service:tempo:backend";
|
||||
interval = "5s";
|
||||
http = "${links.tempo.url}/ready";
|
||||
uid = "P214B5B846CF3925F";
|
||||
inherit (links.tempo) url;
|
||||
type = "tempo";
|
||||
jsonData = {
|
||||
serviceMap.datasourceUid = "PBFA97CFB590B2093"; # prometheus
|
||||
nodeGraph.enabled = true;
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
tempo-ingest-otlp-grpc = {
|
||||
mode = "manual";
|
||||
definition = {
|
||||
name = "tempo-ingest-otlp-grpc";
|
||||
address = links.tempo-otlp-grpc.ipv4;
|
||||
inherit (links.tempo-otlp-grpc) port;
|
||||
checks = [
|
||||
{
|
||||
name = "Tempo Service Status";
|
||||
id = "service:tempo-ingest-otlp-grpc:tempo";
|
||||
alias_service = "tempo";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -18,7 +18,7 @@ in
|
|||
};
|
||||
};
|
||||
services.nextcloud = {
|
||||
package = pkgs.nextcloud30;
|
||||
package = pkgs.nextcloud29;
|
||||
enable = true;
|
||||
https = true;
|
||||
hostName = "storage.${depot.lib.meta.domain}";
|
||||
|
|
|
@ -49,9 +49,7 @@ in
|
|||
user = {
|
||||
destroyAfterDays = 0;
|
||||
create = user: psqlSecret "${genPassword} ${user}" ''
|
||||
SELECT 'CREATE USER ${user}'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${user}')\gexec
|
||||
ALTER USER ${user} PASSWORD '@SECRET@';
|
||||
CREATE USER ${user} PASSWORD '@SECRET@';
|
||||
'';
|
||||
destroy = psqlSecret "printenv OBJECT" ''
|
||||
DROP USER @SECRET@;
|
||||
|
@ -61,11 +59,7 @@ in
|
|||
destroyAfterDays = 30;
|
||||
deps = [ "user" ];
|
||||
create = db: psql ''
|
||||
SELECT 'CREATE DATABASE ${db} OWNER ${cfg.databases.${db}.owner}'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${db}')\gexec
|
||||
'';
|
||||
change = db: psql ''
|
||||
ALTER DATABASE ${db} OWNER TO ${cfg.databases.${db}.owner};
|
||||
CREATE DATABASE ${db} OWNER ${cfg.databases.${db}.owner};
|
||||
'';
|
||||
destroy = psqlSecret "printenv OBJECT" ''
|
||||
DROP DATABASE @SECRET@;
|
||||
|
|
|
@ -4,7 +4,6 @@
|
|||
imports = [
|
||||
./options.nix
|
||||
./incandescence.nix
|
||||
./simulacrum/test-data.nix
|
||||
];
|
||||
|
||||
links = {
|
||||
|
@ -37,10 +36,6 @@
|
|||
PATRONI_REWIND_PASSWORD = default;
|
||||
metricsCredentials.nodes = nodes.worker;
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "consul" "incandescence" "locksmith" ];
|
||||
settings = ./simulacrum/test.nix;
|
||||
};
|
||||
simulacrum.deps = [ "consul" "incandescence" "locksmith" ];
|
||||
};
|
||||
}
|
||||
|
|
|
@ -16,7 +16,6 @@ in
|
|||
};
|
||||
};
|
||||
}));
|
||||
default = {};
|
||||
};
|
||||
users = mkOption {
|
||||
type = attrsOf (submodule ({ ... }: {
|
||||
|
@ -33,7 +32,6 @@ in
|
|||
};
|
||||
};
|
||||
}));
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,19 +0,0 @@
|
|||
{ config, lib, ... }:
|
||||
{
|
||||
patroni = lib.mkIf config.simulacrum {
|
||||
databases = config.lib.forService "patroni" {
|
||||
testdb.owner = "testuser";
|
||||
existingdb.owner = "existinguser";
|
||||
};
|
||||
users = config.lib.forService "patroni" {
|
||||
testuser.locksmith = {
|
||||
nodes = config.services.patroni.nodes.haproxy;
|
||||
format = "pgpass";
|
||||
};
|
||||
existinguser.locksmith = {
|
||||
nodes = config.services.patroni.nodes.haproxy;
|
||||
format = "pgpass";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
|
@ -1,117 +0,0 @@
|
|||
{ cluster, ... }:
|
||||
|
||||
let
|
||||
clusterName = "poseidon";
|
||||
link = cluster.config.links.patroni-pg-access;
|
||||
in
|
||||
{
|
||||
defaults = { depot, pkgs, ... }: {
|
||||
environment.systemPackages = [
|
||||
pkgs.jq
|
||||
depot.packages.postgresql
|
||||
];
|
||||
services.patroni.settings.postgresql.pg_hba = [
|
||||
"host postgres postgres 0.0.0.0/0 trust"
|
||||
];
|
||||
};
|
||||
|
||||
# taken from https://github.com/phfroidmont/nixpkgs/blob/patroni-module/nixos/tests/patroni.nix
|
||||
testScript = ''
|
||||
import json
|
||||
nodeNames = json.loads('${builtins.toJSON cluster.config.services.patroni.nodes.worker}')
|
||||
clientNames = json.loads('${builtins.toJSON cluster.config.services.patroni.nodes.haproxy}')
|
||||
nodes = [ n for n in machines if n.name in nodeNames ]
|
||||
clients = [ n for n in machines if n.name in clientNames ]
|
||||
|
||||
def booted(nodes):
|
||||
return filter(lambda node: node.booted, nodes)
|
||||
|
||||
def wait_for_all_nodes_ready(expected_replicas=2):
|
||||
booted_nodes = booted(nodes)
|
||||
for node in booted_nodes:
|
||||
node.wait_for_unit("patroni.service")
|
||||
print(node.succeed("patronictl list ${clusterName}"))
|
||||
node.wait_until_succeeds(f"[ $(patronictl list -f json ${clusterName} | jq 'length') == {expected_replicas + 1} ]")
|
||||
node.wait_until_succeeds("[ $(patronictl list -f json ${clusterName} | jq 'map(select(.Role | test(\"^Leader$\"))) | map(select(.State | test(\"^running$\"))) | length') == 1 ]")
|
||||
node.wait_until_succeeds(f"[ $(patronictl list -f json ${clusterName} | jq 'map(select(.Role | test(\"^Replica$\"))) | map(select(.State | test(\"^streaming$\"))) | length') == {expected_replicas} ]")
|
||||
print(node.succeed("patronictl list ${clusterName}"))
|
||||
for client in booted(clients):
|
||||
client.wait_until_succeeds("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --command='select 1;'")
|
||||
|
||||
def run_dummy_queries():
|
||||
for client in booted(clients):
|
||||
client.succeed("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='insert into dummy(val) values (101);'")
|
||||
client.succeed("test $(psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='select val from dummy where val = 101;') -eq 101")
|
||||
client.succeed("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='delete from dummy where val = 101;'")
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap a new patroni cluster"):
|
||||
wait_for_all_nodes_ready()
|
||||
|
||||
with subtest("should be able to insert and select"):
|
||||
booted_clients = list(booted(clients))
|
||||
booted_clients[0].succeed("psql -h ${link.ipv4} -p ${link.portStr} -U postgres --command='create table dummy as select * from generate_series(1, 100) as val;'")
|
||||
for client in booted_clients:
|
||||
client.succeed("test $(psql -h ${link.ipv4} -p ${link.portStr} -U postgres --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100")
|
||||
|
||||
with subtest("should restart after all nodes are crashed"):
|
||||
for node in nodes:
|
||||
node.crash()
|
||||
for node in nodes:
|
||||
node.start()
|
||||
wait_for_all_nodes_ready()
|
||||
|
||||
with subtest("should be able to run queries while any one node is crashed"):
|
||||
masterNodeName = nodes[0].succeed("patronictl list -f json ${clusterName} | jq '.[] | select(.Role | test(\"^Leader$\")) | .Member' -r").strip()
|
||||
masterNodeIndex = next((i for i, v in enumerate(nodes) if v.name == masterNodeName))
|
||||
|
||||
# Move master node at the end of the list to avoid multiple failovers (makes the test faster and more consistent)
|
||||
nodes.append(nodes.pop(masterNodeIndex))
|
||||
|
||||
for node in nodes:
|
||||
node.crash()
|
||||
wait_for_all_nodes_ready(1)
|
||||
|
||||
# Execute some queries while a node is down.
|
||||
run_dummy_queries()
|
||||
|
||||
# Restart crashed node.
|
||||
node.start()
|
||||
wait_for_all_nodes_ready()
|
||||
|
||||
# Execute some queries with the node back up.
|
||||
run_dummy_queries()
|
||||
|
||||
with subtest("should create databases and users via incandescence"):
|
||||
for client in clients:
|
||||
client.succeed(f"PGPASSFILE=/run/locksmith/patroni-testuser psql -h ${link.ipv4} -p ${link.portStr} -U testuser -d testdb --command='create table test_table_{client.name} as select * from generate_series(1, 10) as val;'")
|
||||
client.fail("PGPASSFILE=/run/locksmith/patroni-testuser psql -h ${link.ipv4} -p ${link.portStr} -U testuser -d postgres --command='select * from dummy;'")
|
||||
|
||||
with subtest("should take over existing databases and users via incandescence"):
|
||||
for cmd in [
|
||||
"drop database existingdb;",
|
||||
"drop user existinguser;",
|
||||
"create database existingdb owner postgres;",
|
||||
"create user existinguser;"
|
||||
]:
|
||||
clients[0].succeed(f"psql -h ${link.ipv4} -p ${link.portStr} -U postgres --command='{cmd}'")
|
||||
|
||||
for client in clients:
|
||||
client.fail(f"PGPASSFILE=/run/locksmith/patroni-existinguser psql -h ${link.ipv4} -p ${link.portStr} -U existinguser -d existingdb --command='create table test_table_{client.name} as select * from generate_series(1, 10) as val;'")
|
||||
|
||||
consulConfig = json.loads(clients[0].succeed("cat /etc/consul.json"))
|
||||
addr = consulConfig["addresses"]["http"]
|
||||
port = consulConfig["ports"]["http"]
|
||||
setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
|
||||
clients[0].succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/patroni/formulae/database/existingdb")
|
||||
clients[0].succeed(f"{setEnv} consul kv delete --recurse services/incandescence/providers/patroni/formulae/user/existinguser")
|
||||
|
||||
for node in nodes:
|
||||
node.systemctl("restart incandescence-patroni.target")
|
||||
clients[0].succeed("[[ $(psql -h ${link.ipv4} -p ${link.portStr} -U postgres --tuples-only --csv --command=\"SELECT pg_roles.rolname FROM pg_database JOIN pg_roles ON pg_database.datdba = pg_roles.oid WHERE pg_database.datname = 'existingdb'\") == existinguser ]]")
|
||||
for client in clients:
|
||||
client.succeed(f"PGPASSFILE=/run/locksmith/patroni-existinguser psql -h ${link.ipv4} -p ${link.portStr} -U existinguser -d existingdb --command='create table test_table_{client.name} as select * from generate_series(1, 10) as val;'")
|
||||
client.fail("PGPASSFILE=/run/locksmith/patroni-existinguser psql -h ${link.ipv4} -p ${link.portStr} -U existinguser -d postgres --command='select * from dummy;'")
|
||||
'';
|
||||
}
|
|
@ -25,10 +25,6 @@ in
|
|||
"d '${baseDir}' 0700 patroni patroni - -"
|
||||
"d '${walDir}' 0700 patroni patroni - -"
|
||||
];
|
||||
systemd.services.patroni = {
|
||||
requires = [ "consul-ready.service" ];
|
||||
after = [ "consul-ready.service" ];
|
||||
};
|
||||
services.patroni = {
|
||||
enable = true;
|
||||
name = hostName;
|
||||
|
@ -61,7 +57,6 @@ in
|
|||
};
|
||||
use_pg_rewind = true;
|
||||
use_slots = true;
|
||||
synchronous_mode = true;
|
||||
authentication = {
|
||||
replication.username = "patronirep";
|
||||
rewind.username = "patronirew";
|
||||
|
@ -72,8 +67,6 @@ in
|
|||
wal_level = "replica";
|
||||
hot_standby_feedback = "on";
|
||||
unix_socket_directories = "/tmp";
|
||||
synchronous_commit = "on";
|
||||
wal_keep_size = 2048;
|
||||
};
|
||||
pg_hba = [
|
||||
"host replication patronirep ${net} scram-sha-256"
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
{ config, depot, ... }:
|
||||
{ depot, ... }:
|
||||
|
||||
{
|
||||
services.sso = {
|
||||
|
@ -18,12 +18,4 @@
|
|||
login.target = ssoAddr;
|
||||
account.target = ssoAddr;
|
||||
};
|
||||
|
||||
patroni = config.lib.forService "sso" {
|
||||
databases.keycloak = {};
|
||||
users.keycloak.locksmith = {
|
||||
nodes = config.services.sso.nodes.host;
|
||||
format = "raw";
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -8,10 +8,12 @@ in
|
|||
{
|
||||
links.keycloak.protocol = "http";
|
||||
|
||||
services.locksmith.waitForSecrets.keycloak = [
|
||||
"patroni-keycloak"
|
||||
];
|
||||
|
||||
age.secrets.keycloak-dbpass = {
|
||||
file = ../../../secrets/keycloak-dbpass.age;
|
||||
owner = "root";
|
||||
group = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
services.nginx.virtualHosts = {
|
||||
"${login}" = lib.recursiveUpdate (vhosts.proxy kc.url) {
|
||||
locations = {
|
||||
|
@ -34,14 +36,13 @@ in
|
|||
host = patroni.ipv4;
|
||||
inherit (patroni) port;
|
||||
useSSL = false;
|
||||
passwordFile = "/run/locksmith/patroni-keycloak";
|
||||
passwordFile = config.age.secrets.keycloak-dbpass.path;
|
||||
};
|
||||
settings = {
|
||||
http-enabled = true;
|
||||
http-host = kc.ipv4;
|
||||
http-port = kc.port;
|
||||
hostname = login;
|
||||
proxy-headers = "xforwarded";
|
||||
proxy = "edge";
|
||||
# for backcompat, TODO: remove
|
||||
http-relative-path = "/auth";
|
||||
};
|
||||
|
@ -53,7 +54,7 @@ in
|
|||
"-Dotel.traces.exporter=otlp"
|
||||
];
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.ways.ingest-traces-otlp.url;
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
|
||||
OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
|
||||
OTEL_TRACES_SAMPLER_ARG = "0.50";
|
||||
};
|
||||
|
|
|
@ -39,6 +39,7 @@ in
|
|||
./garage.nix
|
||||
./garage-options.nix
|
||||
./garage-layout.nix
|
||||
./incandescence-ascensions.nix
|
||||
] ++ lib.optionals config.simulacrum [
|
||||
./simulacrum/snakeoil-rpc-secret.nix
|
||||
];
|
||||
|
@ -56,8 +57,8 @@ in
|
|||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" "ways" ];
|
||||
settings = ./simulacrum/test.nix;
|
||||
deps = [ "consul" "locksmith" "incandescence" "patroni" "ways" ];
|
||||
settings = ./test.nix;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
@ -36,9 +36,10 @@ in
|
|||
inherit (linkWeb) port;
|
||||
checks = [
|
||||
{
|
||||
name = "Garage Service Status";
|
||||
id = "service:garage-web:garage";
|
||||
alias_service = "garage";
|
||||
name = "Garage Node";
|
||||
id = "service:garage-web:node";
|
||||
interval = "5s";
|
||||
http = "${config.links.garageMetrics.url}/health";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
@ -179,13 +179,9 @@ in
|
|||
key = {
|
||||
destroyAfterDays = 0;
|
||||
create = key: ''
|
||||
if [[ "$(garage key info ${lib.escapeShellArg key} 2>&1 >/dev/null)" == "Error: 0 matching keys" ]]; then
|
||||
# don't print secret key
|
||||
garage key new --name ${lib.escapeShellArg key} >/dev/null
|
||||
echo Key ${lib.escapeShellArg key} was created.
|
||||
else
|
||||
echo "Key already exists, assuming ownership"
|
||||
fi
|
||||
'';
|
||||
destroy = ''
|
||||
garage key delete --yes "$OBJECT"
|
||||
|
@ -200,11 +196,7 @@ in
|
|||
deps = [ "key" ];
|
||||
destroyAfterDays = 30;
|
||||
create = bucket: ''
|
||||
if [[ "$(garage bucket info ${lib.escapeShellArg bucket} 2>&1 >/dev/null)" == "Error: Bucket not found" ]]; then
|
||||
garage bucket create ${lib.escapeShellArg bucket}
|
||||
else
|
||||
echo "Bucket already exists, assuming ownership"
|
||||
fi
|
||||
'';
|
||||
destroy = ''
|
||||
garage bucket delete --yes "$OBJECT"
|
||||
|
|
18
cluster/services/storage/incandescence-ascensions.nix
Normal file
18
cluster/services/storage/incandescence-ascensions.nix
Normal file
|
@ -0,0 +1,18 @@
|
|||
{ config, lib, ... }:
|
||||
|
||||
{
|
||||
system.ascensions = {
|
||||
incandescence-garage = lib.mkIf (config.services.incandescence.providers ? garage) {
|
||||
incantations = i: [
|
||||
(i.runGarage /*bash*/ ''
|
||||
garage bucket list | tail -n +2 | cut -d' ' -f3 | while read bucket; do
|
||||
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/bucket/$1/alive" true''} "$bucket"
|
||||
done
|
||||
garage key list | tail -n +2 | cut -d' ' -f5 | while read key; do
|
||||
${i.runConsul /*bash*/ ''consul kv put "services/incandescence/providers/garage/formulae/key/$1/alive" true''} "$key"
|
||||
done
|
||||
'')
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
|
@ -3,17 +3,12 @@
|
|||
{
|
||||
imports = [
|
||||
./options
|
||||
./simulacrum/test-data.nix
|
||||
];
|
||||
|
||||
services.ways = {
|
||||
nodes.host = config.services.websites.nodes.host;
|
||||
nixos.host = ./host.nix;
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
|
||||
settings = ./simulacrum/test.nix;
|
||||
};
|
||||
simulacrum.deps = [ "nginx" "acme-client" "dns" "certificates" "consul" ];
|
||||
};
|
||||
|
||||
dns.records = lib.mapAttrs'
|
||||
|
|
|
@ -25,14 +25,7 @@ in
|
|||
];
|
||||
locations = lib.mkMerge [
|
||||
{
|
||||
"/" = if cfg.grpc then {
|
||||
extraConfig = ''
|
||||
set $nix_proxy_grpc_target ${cfg.target};
|
||||
grpc_pass $nix_proxy_grpc_target;
|
||||
'';
|
||||
} else {
|
||||
proxyPass = cfg.target;
|
||||
};
|
||||
"/".proxyPass = cfg.target;
|
||||
"${cfg.healthCheckPath}".extraConfig = "access_log off;";
|
||||
}
|
||||
{
|
||||
|
@ -89,11 +82,11 @@ in
|
|||
'') consulServiceWays;
|
||||
in pkgs.writeText "ways-upstreams.ctmpl" (lib.concatStringsSep "\n" (lib.unique upstreams));
|
||||
destination = "/run/consul-template/nginx-ways-upstreams.conf";
|
||||
exec.command = lib.singleton (pkgs.writeShellScript "ways-reload" ''
|
||||
if ${config.systemd.package}/bin/systemctl is-active nginx.service; then
|
||||
exec ${config.services.nginx.package}/bin/nginx -s reload -g 'pid /run/nginx/nginx.pid;'
|
||||
fi
|
||||
'');
|
||||
exec.command = [
|
||||
"${config.services.nginx.package}/bin/nginx"
|
||||
"-s" "reload"
|
||||
"-g" "pid /run/nginx/nginx.pid;"
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
@ -35,12 +35,6 @@ with lib;
|
|||
};
|
||||
};
|
||||
|
||||
grpc = mkOption {
|
||||
description = "Whether this endpoint is a gRPC service.";
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
|
||||
target = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
|
@ -107,7 +101,7 @@ with lib;
|
|||
(lib.mkIf options.consulService.isDefined {
|
||||
useConsul = true;
|
||||
nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
|
||||
target = "${if config.grpc then "grpc" else "http"}://${options.nginxUpstreamName.value}";
|
||||
target = "http://${options.nginxUpstreamName.value}";
|
||||
})
|
||||
(lib.mkIf options.bucket.isDefined {
|
||||
consulService = "garage-web";
|
||||
|
|
|
@ -1,11 +0,0 @@
|
|||
{ config, lib, ... }:
|
||||
{
|
||||
ways = lib.mkIf config.simulacrum {
|
||||
ways-test-simple = config.lib.forService "ways" {
|
||||
target = "http://nowhere";
|
||||
};
|
||||
ways-test-consul = config.lib.forService "ways" {
|
||||
consulService = "ways-test-service";
|
||||
};
|
||||
};
|
||||
}
|
|
@ -1,55 +0,0 @@
|
|||
{ cluster, config, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (cluster._module.specialArgs.depot.lib.meta) domain;
|
||||
in
|
||||
|
||||
{
|
||||
nodes = lib.mkMerge [
|
||||
{
|
||||
nowhere = { pkgs, ... }: {
|
||||
networking.firewall.allowedTCPPorts = [ 8080 ];
|
||||
systemd.services.ways-simple-service = let
|
||||
webroot = pkgs.writeTextDir "example.txt" "hello world";
|
||||
in {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.darkhttpd}/bin/darkhttpd ${webroot} --port 8080";
|
||||
DynamicUser = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
(lib.genAttrs cluster.config.services.ways.nodes.host (lib.const {
|
||||
services.nginx.upstreams.nowhere.servers = {
|
||||
"${(builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address}:8080" = {};
|
||||
};
|
||||
consul.services.ways-test-service = {
|
||||
unit = "consul";
|
||||
mode = "external";
|
||||
definition = {
|
||||
name = "ways-test-service";
|
||||
address = (builtins.head config.nodes.nowhere.networking.interfaces.eth1.ipv4.addresses).address;
|
||||
port = 8080;
|
||||
};
|
||||
};
|
||||
}))
|
||||
];
|
||||
|
||||
testScript = ''
|
||||
import json
|
||||
nodeNames = json.loads('${builtins.toJSON cluster.config.services.ways.nodes.host}')
|
||||
nodes = [ n for n in machines if n.name in nodeNames ]
|
||||
|
||||
start_all()
|
||||
nowhere.wait_for_unit("multi-user.target")
|
||||
for node in nodes:
|
||||
node.wait_for_unit("multi-user.target")
|
||||
|
||||
with subtest("single-target service"):
|
||||
nowhere.succeed("curl -f https://ways-test-simple.${domain}")
|
||||
|
||||
with subtest("consul-managed service"):
|
||||
nowhere.succeed("curl -f https://ways-test-consul.${domain}")
|
||||
'';
|
||||
}
|
|
@ -93,9 +93,5 @@ in
|
|||
nodes = config.services.wireguard.nodes.mesh;
|
||||
shared = false;
|
||||
};
|
||||
simulacrum = {
|
||||
enable = true;
|
||||
settings = ./test.nix;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,28 +0,0 @@
|
|||
{ cluster, lib, ... }:
|
||||
|
||||
{
|
||||
defaults.options.services.locksmith = lib.mkSinkUndeclaredOptions { };
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
${lib.pipe cluster.config.services.wireguard.nodes.mesh [
|
||||
(map (node: /*python*/ ''
|
||||
${node}.wait_for_unit("wireguard-wgmesh.target")
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
|
||||
${lib.pipe cluster.config.services.wireguard.nodes.mesh [
|
||||
(map (node: /*python*/ ''
|
||||
with subtest("${node} can reach all other nodes"):
|
||||
${lib.pipe (cluster.config.services.wireguard.otherNodes.mesh node) [
|
||||
(map (peer: /*python*/ ''
|
||||
${node}.succeed("ping -c3 ${cluster.config.hostLinks.${peer}.mesh.extra.meshIp}")
|
||||
''))
|
||||
(lib.concatStringsSep "\n ")
|
||||
]}
|
||||
''))
|
||||
(lib.concatStringsSep "\n")
|
||||
]}
|
||||
'';
|
||||
}
|
111
flake.lock
111
flake.lock
|
@ -10,11 +10,11 @@
|
|||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1723293904,
|
||||
"narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
|
||||
"lastModified": 1722339003,
|
||||
"narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
|
||||
"owner": "ryantm",
|
||||
"repo": "agenix",
|
||||
"rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
|
||||
"rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -29,8 +29,9 @@
|
|||
"flake-compat": [
|
||||
"blank"
|
||||
],
|
||||
"flake-parts": "flake-parts",
|
||||
"nix-github-actions": "nix-github-actions",
|
||||
"flake-utils": [
|
||||
"repin-flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
|
@ -39,11 +40,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1730906442,
|
||||
"narHash": "sha256-tBuyb8jWBSHHgcIrOfiyQJZGY1IviMzH2V74t7gWfgI=",
|
||||
"lastModified": 1722472866,
|
||||
"narHash": "sha256-GJIz4M5HDB948Ex/8cPvbkrNzl/eKUE7/c21JBu4lb8=",
|
||||
"owner": "zhaofengli",
|
||||
"repo": "attic",
|
||||
"rev": "d0b66cf897e4d55f03d341562c9821dc4e566e54",
|
||||
"rev": "e127acbf9a71ebc0c26bc8e28346822e0a6e16ba",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -75,11 +76,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1722960479,
|
||||
"narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
|
||||
"lastModified": 1717025063,
|
||||
"narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
|
||||
"rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -117,11 +118,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1728330715,
|
||||
"narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
|
||||
"lastModified": 1722113426,
|
||||
"narHash": "sha256-Yo/3loq572A8Su6aY5GP56knpuKYRvM2a1meP9oJZCw=",
|
||||
"owner": "numtide",
|
||||
"repo": "devshell",
|
||||
"rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
|
||||
"rev": "67cce7359e4cd3c45296fb4aaf6a19e2a9c757ae",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -189,7 +190,6 @@
|
|||
"flake-parts": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
"attic",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
|
@ -208,26 +208,6 @@
|
|||
}
|
||||
},
|
||||
"flake-parts_2": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1730504689,
|
||||
"narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "506278e768c2a08bec68eb62932193e341f55c90",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-parts_3": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
"nix-super",
|
||||
|
@ -288,11 +268,11 @@
|
|||
"nixpkgs": "nixpkgs"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1723736589,
|
||||
"narHash": "sha256-/Vdg5ZKtP71ZEKVV6JXlrOEu0CM2Flcs+nwDmWRzgjQ=",
|
||||
"lastModified": 1720223941,
|
||||
"narHash": "sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "hercules-ci-agent",
|
||||
"rev": "c303cc8e437c0fd26b9452472e7df5aa374e9177",
|
||||
"rev": "2e10fb21fc2e07edf40763b73443e5934bd40947",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -311,11 +291,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1730903510,
|
||||
"narHash": "sha256-mnynlrPeiW0nUQ8KGZHb3WyxAxA3Ye/BH8gMjdoKP6E=",
|
||||
"lastModified": 1719226092,
|
||||
"narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "hercules-ci-effects",
|
||||
"rev": "b89ac4d66d618b915b1f0a408e2775fe3821d141",
|
||||
"rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -419,11 +399,11 @@
|
|||
"systems": "systems_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1725623828,
|
||||
"narHash": "sha256-5Zrn72PO9yBaNO4Gd5uOsEmRpYH5rVAFKOQ5h2PxyhU=",
|
||||
"lastModified": 1722409392,
|
||||
"narHash": "sha256-8QuMS00EutmqzAIPxyJEPxM8EHiWlSKs6E2Htoh3Kes=",
|
||||
"owner": "numtide",
|
||||
"repo": "nar-serve",
|
||||
"rev": "e5c749a444f2d14f381c75ef3a8feaa82c333b92",
|
||||
"rev": "9d0eff868d328fe67c60c26c8ba50e0b9d8de867",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -434,11 +414,11 @@
|
|||
},
|
||||
"nix-filter": {
|
||||
"locked": {
|
||||
"lastModified": 1730207686,
|
||||
"narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
|
||||
"lastModified": 1710156097,
|
||||
"narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
|
||||
"owner": "numtide",
|
||||
"repo": "nix-filter",
|
||||
"rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
|
||||
"rev": "3342559a24e85fc164b295c3444e8a139924675b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -447,31 +427,10 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix-github-actions": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"attic",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1729742964,
|
||||
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix-super": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat_2",
|
||||
"flake-parts": "flake-parts_3",
|
||||
"flake-parts": "flake-parts_2",
|
||||
"libgit2": "libgit2",
|
||||
"nixpkgs": "nixpkgs_3",
|
||||
"nixpkgs-regression": [
|
||||
|
@ -545,11 +504,11 @@
|
|||
},
|
||||
"nixpkgs_4": {
|
||||
"locked": {
|
||||
"lastModified": 1730785428,
|
||||
"narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
|
||||
"lastModified": 1722539632,
|
||||
"narHash": "sha256-g4L+I8rDl7RQy5x8XcEMqNO49LFhrHTzVBqXtG2+FGo=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
|
||||
"rev": "f2d6c7123138044e0c68902268bd8f37dd7e2fa7",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -598,11 +557,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1726560853,
|
||||
"narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
|
||||
"lastModified": 1710146030,
|
||||
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
|
||||
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -618,7 +577,7 @@
|
|||
"blank": "blank",
|
||||
"devshell": "devshell",
|
||||
"drv-parts": "drv-parts",
|
||||
"flake-parts": "flake-parts_2",
|
||||
"flake-parts": "flake-parts",
|
||||
"hercules-ci-agent": "hercules-ci-agent",
|
||||
"hercules-ci-effects": "hercules-ci-effects",
|
||||
"hyprspace": "hyprspace",
|
||||
|
|
|
@ -54,6 +54,7 @@
|
|||
nixpkgs.follows = "nixpkgs";
|
||||
nixpkgs-stable.follows = "nixpkgs";
|
||||
flake-compat.follows = "blank";
|
||||
flake-utils.follows = "repin-flake-utils";
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
@ -26,13 +26,11 @@ in
|
|||
|
||||
cfg = v.distributed;
|
||||
|
||||
svcs = map (x: config.consul.services.${x}) cfg.registerServices;
|
||||
svc = config.consul.services.${cfg.registerService};
|
||||
|
||||
runWithRegistration = pkgs.writeShellScript "run-with-registration" ''
|
||||
trap '${lib.concatStringsSep ";" (map (svc: svc.commands.deregister) svcs)}' EXIT
|
||||
${lib.concatStringsSep "\n" (
|
||||
map (svc: svc.commands.register) svcs
|
||||
)}
|
||||
trap '${svc.commands.deregister}' EXIT
|
||||
${svc.commands.register}
|
||||
''${@}
|
||||
'';
|
||||
|
||||
|
@ -51,10 +49,10 @@ in
|
|||
[Service]
|
||||
ExecStartPre=${waitForConsul} 'services/${n}%i'
|
||||
ExecStart=
|
||||
ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerServices != []) runWithRegistration} ${ExecStart}
|
||||
ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
|
||||
Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
|
||||
${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
|
||||
${optionalString (cfg.registerServices != []) (lib.concatStringsSep "\n" (map (svc: "ExecStopPost=${svc.commands.deregister}") svcs))}
|
||||
${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
|
||||
''))
|
||||
];
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
{ cluster, config, depot, pkgs, ... }:
|
||||
{ config, depot, pkgs, ... }:
|
||||
{
|
||||
users.motd = builtins.readFile ./motd.txt;
|
||||
environment.interactiveShellInit = let
|
||||
|
@ -8,11 +8,6 @@
|
|||
grep = exec pkgs.gnugrep "grep";
|
||||
countUsers = '' ${util "who"} -q | ${util "head"} -n1 | ${util "tr"} ' ' \\n | ${util "uniq"} | ${util "wc"} -l'';
|
||||
countSessions = '' ${util "who"} -q | ${util "head"} -n1 | ${util "wc"} -w'';
|
||||
|
||||
rev = if cluster.config.simulacrum then
|
||||
"simulacrum"
|
||||
else
|
||||
depot.rev or "\${BRED}(✘)\${CO}\${BWHITE} Dirty";
|
||||
in ''
|
||||
(
|
||||
# Reset colors
|
||||
|
@ -45,7 +40,7 @@
|
|||
echo -e " █ ''${BGREEN}(✓)''${CO} ''${BWHITE}You are using a genuine Private Void™ system.''${CO}"
|
||||
echo " █"
|
||||
echo -e " █ ''${BWHITE}OS Version....:''${CO} NixOS ''${CAB}${config.system.nixos.version}''${CO}"
|
||||
echo -e " █ ''${BWHITE}Configuration.:''${CO} ''${CAB}${rev}''${CO}"
|
||||
echo -e " █ ''${BWHITE}Configuration.:''${CO} ''${CAB}${depot.rev or "\${BRED}(✘)\${CO}\${BWHITE} Dirty"}''${CO}"
|
||||
echo -e " █ ''${BWHITE}Uptime........:''${CO} $(${uptime} -p | ${util "cut"} -d ' ' -f2- | GREP_COLORS='mt=01;35' ${grep} --color=always '[0-9]*')"
|
||||
echo -e " █ ''${BWHITE}SSH Logins....:''${CO} There are currently ''${CAB}$(${countUsers})''${CO} users logged in on ''${CAB}$(${countSessions})''${CO} sessions"
|
||||
)
|
||||
|
|
|
@ -29,6 +29,7 @@ in
|
|||
reflection = ./reflection;
|
||||
shell-config = ./shell-config;
|
||||
ssh = ./ssh;
|
||||
system-info = ./system-info;
|
||||
system-recovery = ./system-recovery;
|
||||
systemd-extras = ./systemd-extras;
|
||||
tested = ./tested;
|
||||
|
@ -55,6 +56,7 @@ in
|
|||
motd
|
||||
networking
|
||||
nix-config-server
|
||||
system-info
|
||||
system-recovery
|
||||
tested
|
||||
];
|
||||
|
|
5
modules/system-info/default.nix
Normal file
5
modules/system-info/default.nix
Normal file
|
@ -0,0 +1,5 @@
|
|||
{ depot, ... }:
|
||||
|
||||
{
|
||||
system.configurationRevision = depot.rev or null;
|
||||
}
|
|
@ -17,11 +17,6 @@ with lib;
|
|||
type = with types; nullOr str;
|
||||
default = null;
|
||||
};
|
||||
registerServices = mkOption {
|
||||
description = "Consul services to register when this service gets started.";
|
||||
type = with types; listOf str;
|
||||
default = if config.distributed.registerService == null then [ ] else [ config.distributed.registerService ];
|
||||
};
|
||||
};
|
||||
}));
|
||||
};
|
||||
|
|
|
@ -7,6 +7,7 @@ let
|
|||
in
|
||||
|
||||
{
|
||||
debug = lib.warn "debug mode is enabled" true;
|
||||
perSystem = { filters, pkgs, self', system, ... }: {
|
||||
checks = lib.mkIf (system == "x86_64-linux") {
|
||||
ascensions = pkgs.callPackage ./ascensions.nix {
|
||||
|
@ -29,6 +30,11 @@ in
|
|||
inherit (self'.packages) keycloak;
|
||||
};
|
||||
|
||||
patroni = pkgs.callPackage ./patroni.nix {
|
||||
inherit (self) nixosModules;
|
||||
inherit (self'.packages) postgresql;
|
||||
};
|
||||
|
||||
s3ql-upgrade = pkgs.callPackage ./s3ql-upgrade.nix {
|
||||
inherit (self'.packages) s3ql;
|
||||
inherit (self) nixosModules;
|
||||
|
|
|
@ -7,8 +7,7 @@ nixosTest {
|
|||
package = keycloak;
|
||||
database.passwordFile = builtins.toFile "keycloak-test-password" "kcnixostest1234";
|
||||
settings = {
|
||||
http-enabled = true;
|
||||
proxy-headers = "xforwarded";
|
||||
proxy = "edge";
|
||||
hostname = "keycloak.local";
|
||||
};
|
||||
};
|
||||
|
|
211
packages/checks/patroni.nix
Normal file
211
packages/checks/patroni.nix
Normal file
|
@ -0,0 +1,211 @@
|
|||
{ nixosTest, nixosModules, postgresql }:
|
||||
|
||||
# taken from https://github.com/phfroidmont/nixpkgs/blob/patroni-module/nixos/tests/patroni.nix
|
||||
nixosTest (
|
||||
let
|
||||
nodesIps = [
|
||||
"192.168.1.1"
|
||||
"192.168.1.2"
|
||||
"192.168.1.3"
|
||||
];
|
||||
|
||||
createNode = index: { pkgs, ... }:
|
||||
let
|
||||
ip = builtins.elemAt nodesIps index; # since we already use IPs to identify servers
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
nixosModules.patroni
|
||||
nixosModules.systemd-extras
|
||||
];
|
||||
|
||||
networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
|
||||
{ address = ip; prefixLength = 16; }
|
||||
];
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 5432 8008 5010 ];
|
||||
|
||||
environment.systemPackages = [ pkgs.jq ];
|
||||
|
||||
services.patroni = {
|
||||
|
||||
enable = true;
|
||||
|
||||
postgresqlPackage = postgresql.withPackages (p: [ p.pg_safeupdate ]);
|
||||
|
||||
scope = "cluster1";
|
||||
name = "node${toString(index + 1)}";
|
||||
nodeIp = ip;
|
||||
otherNodesIps = builtins.filter (h: h != ip) nodesIps;
|
||||
softwareWatchdog = true;
|
||||
|
||||
settings = {
|
||||
bootstrap = {
|
||||
dcs = {
|
||||
ttl = 30;
|
||||
loop_wait = 10;
|
||||
retry_timeout = 10;
|
||||
maximum_lag_on_failover = 1048576;
|
||||
};
|
||||
initdb = [
|
||||
{ encoding = "UTF8"; }
|
||||
"data-checksums"
|
||||
];
|
||||
};
|
||||
|
||||
postgresql = {
|
||||
use_pg_rewind = true;
|
||||
use_slots = true;
|
||||
authentication = {
|
||||
replication = {
|
||||
username = "replicator";
|
||||
};
|
||||
superuser = {
|
||||
username = "postgres";
|
||||
};
|
||||
rewind = {
|
||||
username = "rewind";
|
||||
};
|
||||
};
|
||||
parameters = {
|
||||
listen_addresses = "${ip}";
|
||||
wal_level = "replica";
|
||||
hot_standby_feedback = "on";
|
||||
unix_socket_directories = "/tmp";
|
||||
};
|
||||
pg_hba = [
|
||||
"host replication replicator 192.168.1.0/24 md5"
|
||||
# Unsafe, do not use for anything other than tests
|
||||
"host all all 0.0.0.0/0 trust"
|
||||
];
|
||||
};
|
||||
|
||||
etcd3 = {
|
||||
host = "192.168.1.4:2379";
|
||||
};
|
||||
};
|
||||
|
||||
environmentFiles = {
|
||||
PATRONI_REPLICATION_PASSWORD = pkgs.writeText "replication-password" "postgres";
|
||||
PATRONI_SUPERUSER_PASSWORD = pkgs.writeText "superuser-password" "postgres";
|
||||
PATRONI_REWIND_PASSWORD = pkgs.writeText "rewind-password" "postgres";
|
||||
};
|
||||
};
|
||||
|
||||
# We always want to restart so the tests never hang
|
||||
systemd.services.patroni.serviceConfig.StartLimitIntervalSec = 0;
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "patroni";
|
||||
|
||||
nodes = {
|
||||
node1 = createNode 0;
|
||||
node2 = createNode 1;
|
||||
node3 = createNode 2;
|
||||
|
||||
etcd = { pkgs, ... }: {
|
||||
|
||||
networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
|
||||
{ address = "192.168.1.4"; prefixLength = 16; }
|
||||
];
|
||||
|
||||
services.etcd = {
|
||||
enable = true;
|
||||
listenClientUrls = [ "http://192.168.1.4:2379" ];
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 2379 ];
|
||||
};
|
||||
|
||||
client = { pkgs, ... }: {
|
||||
environment.systemPackages = [ postgresql ];
|
||||
|
||||
networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
|
||||
{ address = "192.168.2.1"; prefixLength = 16; }
|
||||
];
|
||||
|
||||
services.haproxy = {
|
||||
enable = true;
|
||||
config = ''
|
||||
global
|
||||
maxconn 100
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode tcp
|
||||
retries 2
|
||||
timeout client 30m
|
||||
timeout connect 4s
|
||||
timeout server 30m
|
||||
timeout check 5s
|
||||
|
||||
listen cluster1
|
||||
bind 127.0.0.1:5432
|
||||
option httpchk
|
||||
http-check expect status 200
|
||||
default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
|
||||
${builtins.concatStringsSep "\n" (map (ip: "server postgresql_${ip}_5432 ${ip}:5432 maxconn 100 check port 8008") nodesIps)}
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
|
||||
testScript = ''
|
||||
nodes = [node1, node2, node3]
|
||||
|
||||
def wait_for_all_nodes_ready(expected_replicas=2):
|
||||
booted_nodes = filter(lambda node: node.booted, nodes)
|
||||
for node in booted_nodes:
|
||||
print(node.succeed("patronictl list cluster1"))
|
||||
node.wait_until_succeeds(f"[ $(patronictl list -f json cluster1 | jq 'length') == {expected_replicas + 1} ]")
|
||||
node.wait_until_succeeds("[ $(patronictl list -f json cluster1 | jq 'map(select(.Role | test(\"^Leader$\"))) | map(select(.State | test(\"^running$\"))) | length') == 1 ]")
|
||||
node.wait_until_succeeds(f"[ $(patronictl list -f json cluster1 | jq 'map(select(.Role | test(\"^Replica$\"))) | map(select(.State | test(\"^streaming$\"))) | length') == {expected_replicas} ]")
|
||||
print(node.succeed("patronictl list cluster1"))
|
||||
client.wait_until_succeeds("psql -h 127.0.0.1 -U postgres --command='select 1;'")
|
||||
|
||||
def run_dummy_queries():
|
||||
client.succeed("psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='insert into dummy(val) values (101);'")
|
||||
client.succeed("test $(psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='select val from dummy where val = 101;') -eq 101")
|
||||
client.succeed("psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='delete from dummy where val = 101;'")
|
||||
|
||||
start_all()
|
||||
|
||||
with subtest("should bootstrap a new patroni cluster"):
|
||||
wait_for_all_nodes_ready()
|
||||
|
||||
with subtest("should be able to insert and select"):
|
||||
client.succeed("psql -h 127.0.0.1 -U postgres --command='create table dummy as select * from generate_series(1, 100) as val;'")
|
||||
client.succeed("test $(psql -h 127.0.0.1 -U postgres --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100")
|
||||
|
||||
with subtest("should restart after all nodes are crashed"):
|
||||
for node in nodes:
|
||||
node.crash()
|
||||
for node in nodes:
|
||||
node.start()
|
||||
wait_for_all_nodes_ready()
|
||||
|
||||
with subtest("should be able to run queries while any one node is crashed"):
|
||||
masterNodeName = node1.succeed("patronictl list -f json cluster1 | jq '.[] | select(.Role | test(\"^Leader$\")) | .Member' -r").strip()
|
||||
masterNodeIndex = int(masterNodeName[len(masterNodeName)-1]) - 1
|
||||
|
||||
# Move master node at the end of the list to avoid multiple failovers (makes the test faster and more consistent)
|
||||
nodes.append(nodes.pop(masterNodeIndex))
|
||||
|
||||
for node in nodes:
|
||||
node.crash()
|
||||
wait_for_all_nodes_ready(1)
|
||||
|
||||
# Execute some queries while a node is down.
|
||||
run_dummy_queries()
|
||||
|
||||
# Restart crashed node.
|
||||
node.start()
|
||||
wait_for_all_nodes_ready()
|
||||
|
||||
# Execute some queries with the node back up.
|
||||
run_dummy_queries()
|
||||
'';
|
||||
})
|
|
@ -10,9 +10,6 @@ testers.runNixOSTest {
|
|||
nixosModules.systemd-extras
|
||||
./modules/nixos/age-dummy-secrets
|
||||
./modules/nixos/age-dummy-secrets/options.nix
|
||||
{
|
||||
options.services.locksmith = lib.mkSinkUndeclaredOptions { };
|
||||
}
|
||||
];
|
||||
|
||||
_module.args.depot.packages = { inherit (previous.packages.${system}) s3ql; };
|
||||
|
|
|
@ -18,10 +18,6 @@ super: rec {
|
|||
};
|
||||
};
|
||||
|
||||
jitsi-meet-insecure = let
|
||||
olm-insecure = assert builtins.length super.olm.meta.knownVulnerabilities > 0; super.olm.overrideAttrs (o: { meta = o.meta // { knownVulnerabilities = []; }; });
|
||||
in super.jitsi-meet.override { olm = olm-insecure; };
|
||||
|
||||
jre17_standard = let
|
||||
jre = super.jre_minimal.override {
|
||||
jdk = super.jdk17_headless;
|
||||
|
@ -50,7 +46,7 @@ super: rec {
|
|||
|
||||
prometheus-jitsi-exporter = patch super.prometheus-jitsi-exporter "patches/base/prometheus-jitsi-exporter";
|
||||
|
||||
s3ql = super.s3ql.overrideAttrs (old: {
|
||||
s3ql = (patch super.s3ql "patches/base/s3ql").overrideAttrs (old: {
|
||||
propagatedBuildInputs = old.propagatedBuildInputs ++ [
|
||||
super.python3Packages.packaging
|
||||
super.python3Packages.systemd
|
||||
|
|
|
@ -11,6 +11,8 @@
|
|||
nix-super = packages.nix-super.nix;
|
||||
|
||||
agenix = packages.agenix.agenix.override { nix = nix-super; };
|
||||
|
||||
hci = packages.hercules-ci-agent.hercules-ci-cli;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -59,6 +59,7 @@
|
|||
tools = with flakePkgs; [
|
||||
agenix
|
||||
graf
|
||||
hci
|
||||
npins
|
||||
pin
|
||||
void
|
||||
|
|
|
@ -9,9 +9,7 @@
|
|||
|
||||
options.shadows = lib.mkOption {
|
||||
type = with lib.types; lazyAttrsOf package;
|
||||
default = {
|
||||
jitsi-meet = self'.packages.jitsi-meet-insecure;
|
||||
};
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
{
|
||||
packages = {
|
||||
hci = [ "x86_64-linux" ];
|
||||
hydra = [ "x86_64-linux" ];
|
||||
jellyfin = [ "x86_64-linux" ];
|
||||
keycloak = [ "x86_64-linux" ];
|
||||
|
|
|
@ -1,28 +1,25 @@
|
|||
diff --git a/unix_integration/resolver/src/idprovider/kanidm.rs b/unix_integration/resolver/src/idprovider/kanidm.rs
|
||||
index 63cedb4d5..35c45fb0e 100644
|
||||
--- a/unix_integration/resolver/src/idprovider/kanidm.rs
|
||||
+++ b/unix_integration/resolver/src/idprovider/kanidm.rs
|
||||
@@ -7,6 +7,7 @@ use kanidm_proto::internal::OperationError;
|
||||
diff --git a/unix_integration/src/idprovider/kanidm.rs b/unix_integration/src/idprovider/kanidm.rs
|
||||
index 6fc015756..31593f03e 100644
|
||||
--- a/unix_integration/src/idprovider/kanidm.rs
|
||||
+++ b/unix_integration/src/idprovider/kanidm.rs
|
||||
@@ -4,6 +4,7 @@ use kanidm_client::{ClientError, KanidmClient, StatusCode};
|
||||
use kanidm_proto::internal::OperationError;
|
||||
use kanidm_proto::v1::{UnixGroupToken, UnixUserToken};
|
||||
use std::collections::BTreeSet;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::{broadcast, RwLock};
|
||||
+use std::env;
|
||||
use tokio::sync::{broadcast, Mutex};
|
||||
|
||||
use kanidm_lib_crypto::CryptoPolicy;
|
||||
@@ -38,6 +39,8 @@ struct KanidmProviderInternal {
|
||||
hmac_key: HmacKey,
|
||||
crypto_policy: CryptoPolicy,
|
||||
pam_allow_groups: BTreeSet<String>,
|
||||
use super::interface::{
|
||||
// KeyStore,
|
||||
@@ -25,12 +26,28 @@ const TAG_IDKEY: &str = "idkey";
|
||||
|
||||
pub struct KanidmProvider {
|
||||
client: RwLock<KanidmClient>,
|
||||
+ auth_name: Option<String>,
|
||||
+ auth_password: Option<String>,
|
||||
}
|
||||
|
||||
pub struct KanidmProvider {
|
||||
@@ -102,6 +105,19 @@ impl KanidmProvider {
|
||||
.map(|GroupMap { local, with }| (local, Id::Name(with)))
|
||||
.collect();
|
||||
|
||||
impl KanidmProvider {
|
||||
pub fn new(client: KanidmClient) -> Self {
|
||||
+ let env_username: Option<String>;
|
||||
+ let env_password: Option<String>;
|
||||
+ match (env::var_os("KANIDM_NAME"), env::var_os("KANIDM_PASSWORD")) {
|
||||
|
@ -35,29 +32,23 @@ index 63cedb4d5..35c45fb0e 100644
|
|||
+ env_password = None;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
Ok(KanidmProvider {
|
||||
inner: Mutex::new(KanidmProviderInternal {
|
||||
state: CacheState::OfflineNextCheck(now),
|
||||
@@ -109,6 +125,8 @@ impl KanidmProvider {
|
||||
hmac_key,
|
||||
crypto_policy,
|
||||
pam_allow_groups,
|
||||
KanidmProvider {
|
||||
client: RwLock::new(client),
|
||||
+ auth_name: env_username,
|
||||
+ auth_password: env_password
|
||||
}),
|
||||
map_group,
|
||||
})
|
||||
@@ -256,7 +274,11 @@ impl KanidmProviderInternal {
|
||||
+ auth_password: env_password,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,7 +135,11 @@ impl IdProvider for KanidmProvider {
|
||||
|
||||
async fn attempt_online(&mut self, _tpm: &mut tpm::BoxedDynTpm, now: SystemTime) -> bool {
|
||||
- match self.client.auth_anonymous().await {
|
||||
// Needs .read on all types except re-auth.
|
||||
async fn provider_authenticate(&self, _tpm: &mut tpm::BoxedDynTpm) -> Result<(), IdpError> {
|
||||
- match self.client.write().await.auth_anonymous().await {
|
||||
+ let auth_method = match (&self.auth_name, &self.auth_password) {
|
||||
+ (Some(name), Some(password)) => self.client.auth_simple_password(name, password).await,
|
||||
+ _ => self.client.auth_anonymous().await
|
||||
+ (Some(name), Some(password)) => self.client.write().await.auth_simple_password(name, password).await,
|
||||
+ _ => self.client.write().await.auth_anonymous().await
|
||||
+ };
|
||||
+ match auth_method {
|
||||
Ok(_uat) => {
|
||||
self.state = CacheState::Online;
|
||||
true
|
||||
Ok(_uat) => Ok(()),
|
||||
Err(err) => {
|
||||
error!(?err, "Provider authentication failed");
|
||||
|
|
18
patches/base/s3ql/0000-cache-entry-seek-whence.patch
Normal file
18
patches/base/s3ql/0000-cache-entry-seek-whence.patch
Normal file
|
@ -0,0 +1,18 @@
|
|||
diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py
|
||||
index a4b55fd1..267b9a12 100644
|
||||
--- a/src/s3ql/block_cache.py
|
||||
+++ b/src/s3ql/block_cache.py
|
||||
@@ -86,10 +86,10 @@ class CacheEntry:
|
||||
def flush(self):
|
||||
self.fh.flush()
|
||||
|
||||
- def seek(self, off):
|
||||
+ def seek(self, off, whence=0):
|
||||
if self.pos != off:
|
||||
- self.fh.seek(off)
|
||||
- self.pos = off
|
||||
+ self.fh.seek(off, whence)
|
||||
+ self.pos = self.fh.tell()
|
||||
|
||||
def tell(self):
|
||||
return self.pos
|
26
patches/base/s3ql/0001-fix-plain-block-size.patch
Normal file
26
patches/base/s3ql/0001-fix-plain-block-size.patch
Normal file
|
@ -0,0 +1,26 @@
|
|||
diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
|
||||
index 6402fec1..9ed3627e 100644
|
||||
--- a/src/s3ql/backends/comprenc.py
|
||||
+++ b/src/s3ql/backends/comprenc.py
|
||||
@@ -276,7 +276,7 @@ class ComprencBackend(AbstractBackend):
|
||||
buf.seek(0)
|
||||
fh = buf
|
||||
|
||||
- return self.backend.write_fh(key, fh, meta_raw)
|
||||
+ return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
|
||||
|
||||
def contains(self, key):
|
||||
return self.backend.contains(key)
|
||||
diff --git a/src/s3ql/database.py b/src/s3ql/database.py
|
||||
index bb4054e6..c2142bf6 100644
|
||||
--- a/src/s3ql/database.py
|
||||
+++ b/src/s3ql/database.py
|
||||
@@ -659,7 +659,7 @@ def upload_metadata(
|
||||
)
|
||||
obj = METADATA_OBJ_NAME % (blockno, params.seq_no)
|
||||
fh.seek(blockno * blocksize)
|
||||
- backend.write_fh(obj, fh, len_=blocksize)
|
||||
+ backend.write_fh(obj, fh, len_=min(blocksize, db_size - blockno * blocksize))
|
||||
|
||||
if not update_params:
|
||||
return
|
17
patches/base/s3ql/0002-comprenc-always-copy.patch
Normal file
17
patches/base/s3ql/0002-comprenc-always-copy.patch
Normal file
|
@ -0,0 +1,17 @@
|
|||
diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
|
||||
index 9ed3627e..db419bb7 100644
|
||||
--- a/src/s3ql/backends/comprenc.py
|
||||
+++ b/src/s3ql/backends/comprenc.py
|
||||
@@ -276,6 +276,12 @@ class ComprencBackend(AbstractBackend):
|
||||
buf.seek(0)
|
||||
fh = buf
|
||||
|
||||
+ if meta_raw['compression'] == 'None' and meta_raw['encryption'] == 'None':
|
||||
+ buf = io.BytesIO()
|
||||
+ copyfh(fh, buf, len_)
|
||||
+ buf.seek(0)
|
||||
+ fh = buf
|
||||
+
|
||||
return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
|
||||
|
||||
def contains(self, key):
|
12
patches/base/s3ql/remove-ssl-monkeypatch.patch
Normal file
12
patches/base/s3ql/remove-ssl-monkeypatch.patch
Normal file
|
@ -0,0 +1,12 @@
|
|||
diff --git a/tests/t0_http.py b/tests/t0_http.py
|
||||
index 66ed564f..36bebab1 100755
|
||||
--- a/tests/t0_http.py
|
||||
+++ b/tests/t0_http.py
|
||||
@@ -289,7 +289,6 @@ def do_GET(self):
|
||||
|
||||
# We don't *actually* want to establish SSL, that'd be
|
||||
# to complex for our mock server
|
||||
- monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
|
||||
conn = HTTPConnection(
|
||||
test_host,
|
||||
test_port,
|
26
patches/base/s3ql/s3c-accurate-length.patch
Normal file
26
patches/base/s3ql/s3c-accurate-length.patch
Normal file
|
@ -0,0 +1,26 @@
|
|||
commit 1edbbcf08d5701ea38f13fca7491418318aebca9
|
||||
Author: Max <max@privatevoid.net>
|
||||
Date: Fri Jun 7 23:31:08 2024 +0200
|
||||
|
||||
accurate length
|
||||
|
||||
diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py
|
||||
index 2995ca4f..3c3c79ab 100644
|
||||
--- a/src/s3ql/backends/s3c.py
|
||||
+++ b/src/s3ql/backends/s3c.py
|
||||
@@ -387,9 +387,13 @@ class Backend(AbstractBackend):
|
||||
'''
|
||||
|
||||
off = fh.tell()
|
||||
+ fh.seek(0, os.SEEK_END)
|
||||
+ actual_len = fh.tell() - off
|
||||
+ fh.seek(off, os.SEEK_SET)
|
||||
if len_ is None:
|
||||
- fh.seek(0, os.SEEK_END)
|
||||
- len_ = fh.tell()
|
||||
+ len_ = actual_len
|
||||
+ else:
|
||||
+ len_ = min(len_, actual_len)
|
||||
return self._write_fh(key, fh, off, len_, metadata or {})
|
||||
|
||||
@retry
|
392
patches/base/s3ql/s3v4.patch
Normal file
392
patches/base/s3ql/s3v4.patch
Normal file
|
@ -0,0 +1,392 @@
|
|||
From 11e3a9cea77cd8498d874f7fd69a938af4da68cd Mon Sep 17 00:00:00 2001
|
||||
From: xeji <36407913+xeji@users.noreply.github.com>
|
||||
Date: Thu, 28 Mar 2024 22:19:11 +0100
|
||||
Subject: [PATCH] new backend s3c4: s3c with V4 request signatures (#349)
|
||||
|
||||
---
|
||||
rst/backends.rst | 15 ++++
|
||||
src/s3ql/backends/__init__.py | 3 +-
|
||||
src/s3ql/backends/s3.py | 100 ++----------------------
|
||||
src/s3ql/backends/s3c4.py | 140 ++++++++++++++++++++++++++++++++++
|
||||
src/s3ql/parse_args.py | 2 +-
|
||||
tests/mock_server.py | 11 +++
|
||||
6 files changed, 174 insertions(+), 97 deletions(-)
|
||||
create mode 100644 src/s3ql/backends/s3c4.py
|
||||
|
||||
diff --git a/rst/backends.rst b/rst/backends.rst
|
||||
index 7220ee96..4bc68387 100644
|
||||
--- a/rst/backends.rst
|
||||
+++ b/rst/backends.rst
|
||||
@@ -341,6 +341,14 @@ can be an arbitrary prefix that will be prepended to all object names
|
||||
used by S3QL. This allows you to store several S3QL file systems in
|
||||
the same bucket.
|
||||
|
||||
+`s3c://` authenticates API requests using AWS V2 signatures, which are
|
||||
+deprecated by AWS but still accepted by many S3 compatible services.
|
||||
+
|
||||
+`s3c4://` denotes a variant of this backend that works the same
|
||||
+but uses AWS V4 signatures for request authentication instead: ::
|
||||
+
|
||||
+ s3c4://<hostname>:<port>/<bucketname>/<prefix>
|
||||
+
|
||||
The S3 compatible backend accepts the following backend options:
|
||||
|
||||
.. option:: no-ssl
|
||||
@@ -385,6 +393,13 @@ The S3 compatible backend accepts the following backend options:
|
||||
necessary if your storage server does not return a valid response
|
||||
body for a successful copy operation.
|
||||
|
||||
+.. option:: sig-region=<region>
|
||||
+
|
||||
+ For `s3c4://` variant only: Region to use for calculating V4
|
||||
+ request signatures. Contrary to S3, the region is not a defined
|
||||
+ part of the storage URL and must be specified separately.
|
||||
+ Defaults to `us-east-1`.
|
||||
+
|
||||
.. _`S3 COPY API`: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
|
||||
.. __: https://doc.s3.amazonaws.com/proposals/copy.html
|
||||
|
||||
diff --git a/src/s3ql/backends/__init__.py b/src/s3ql/backends/__init__.py
|
||||
index a1335762..442828cd 100644
|
||||
--- a/src/s3ql/backends/__init__.py
|
||||
+++ b/src/s3ql/backends/__init__.py
|
||||
@@ -6,7 +6,7 @@
|
||||
This work can be distributed under the terms of the GNU GPLv3.
|
||||
'''
|
||||
|
||||
-from . import gs, local, rackspace, s3, s3c, swift, swiftks
|
||||
+from . import gs, local, rackspace, s3, s3c, s3c4, swift, swiftks
|
||||
from .b2.b2_backend import B2Backend
|
||||
|
||||
#: Mapping from storage URL prefixes to backend classes
|
||||
@@ -15,6 +15,7 @@
|
||||
'local': local.Backend,
|
||||
'gs': gs.Backend,
|
||||
's3c': s3c.Backend,
|
||||
+ 's3c4': s3c4.Backend,
|
||||
'swift': swift.Backend,
|
||||
'swiftks': swiftks.Backend,
|
||||
'rackspace': rackspace.Backend,
|
||||
diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py
|
||||
index e05a49ba..5548a855 100644
|
||||
--- a/src/s3ql/backends/s3.py
|
||||
+++ b/src/s3ql/backends/s3.py
|
||||
@@ -15,7 +15,7 @@
|
||||
from xml.sax.saxutils import escape as xml_escape
|
||||
|
||||
from ..logging import QuietError
|
||||
-from . import s3c
|
||||
+from . import s3c4
|
||||
from .common import retry
|
||||
from .s3c import get_S3Error
|
||||
|
||||
@@ -28,22 +28,23 @@
|
||||
# pylint: disable=E1002,E1101
|
||||
|
||||
|
||||
-class Backend(s3c.Backend):
|
||||
+class Backend(s3c4.Backend):
|
||||
"""A backend to store data in Amazon S3
|
||||
|
||||
This class uses standard HTTP connections to connect to S3.
|
||||
"""
|
||||
|
||||
- known_options = (s3c.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
|
||||
+ known_options = (s3c4.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
|
||||
'dumb-copy',
|
||||
'disable-expect100',
|
||||
+ 'sig-region',
|
||||
}
|
||||
|
||||
def __init__(self, options):
|
||||
self.region = None
|
||||
- self.signing_key = None
|
||||
super().__init__(options)
|
||||
self._set_storage_options(self._extra_put_headers)
|
||||
+ self.sig_region = self.region
|
||||
|
||||
def _parse_storage_url(self, storage_url, ssl_context):
|
||||
hit = re.match(r'^s3s?://([^/]+)/([^/]+)(?:/(.*))?$', storage_url)
|
||||
@@ -147,94 +148,3 @@ def _delete_multi(self, keys):
|
||||
|
||||
except:
|
||||
self.conn.discard()
|
||||
-
|
||||
- def _authorize_request(self, method, path, headers, subres, query_string):
|
||||
- '''Add authorization information to *headers*'''
|
||||
-
|
||||
- # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
|
||||
-
|
||||
- now = time.gmtime()
|
||||
- # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
|
||||
- # '%a, %d %b %Y %H:%M:%S GMT')
|
||||
-
|
||||
- ymd = time.strftime('%Y%m%d', now)
|
||||
- ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
|
||||
-
|
||||
- headers['x-amz-date'] = ymdhms
|
||||
- headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
|
||||
- # headers['x-amz-content-sha256'] = hashlib.sha256(body).hexdigest()
|
||||
- headers.pop('Authorization', None)
|
||||
-
|
||||
- auth_strs = [method]
|
||||
- auth_strs.append(urllib.parse.quote(path))
|
||||
-
|
||||
- if query_string:
|
||||
- s = urllib.parse.urlencode(
|
||||
- query_string, doseq=True, quote_via=urllib.parse.quote
|
||||
- ).split('&')
|
||||
- else:
|
||||
- s = []
|
||||
- if subres:
|
||||
- s.append(urllib.parse.quote(subres) + '=')
|
||||
- if s:
|
||||
- s = '&'.join(sorted(s))
|
||||
- else:
|
||||
- s = ''
|
||||
- auth_strs.append(s)
|
||||
-
|
||||
- # Headers
|
||||
- sig_hdrs = sorted(x.lower() for x in headers.keys())
|
||||
- for hdr in sig_hdrs:
|
||||
- auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
|
||||
- auth_strs.append('')
|
||||
- auth_strs.append(';'.join(sig_hdrs))
|
||||
- auth_strs.append(headers['x-amz-content-sha256'])
|
||||
- can_req = '\n'.join(auth_strs)
|
||||
- # log.debug('canonical request: %s', can_req)
|
||||
-
|
||||
- can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
|
||||
- str_to_sign = (
|
||||
- "AWS4-HMAC-SHA256\n"
|
||||
- + ymdhms
|
||||
- + '\n'
|
||||
- + '%s/%s/s3/aws4_request\n' % (ymd, self.region)
|
||||
- + can_req_hash
|
||||
- )
|
||||
- # log.debug('string to sign: %s', str_to_sign)
|
||||
-
|
||||
- if self.signing_key is None or self.signing_key[1] != ymd:
|
||||
- self.update_signing_key(ymd)
|
||||
- signing_key = self.signing_key[0]
|
||||
-
|
||||
- sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
|
||||
-
|
||||
- cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
|
||||
- self.login,
|
||||
- now.tm_year,
|
||||
- now.tm_mon,
|
||||
- now.tm_mday,
|
||||
- self.region,
|
||||
- )
|
||||
-
|
||||
- headers['Authorization'] = (
|
||||
- 'AWS4-HMAC-SHA256 '
|
||||
- 'Credential=%s,'
|
||||
- 'SignedHeaders=%s,'
|
||||
- 'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
|
||||
- )
|
||||
-
|
||||
- def update_signing_key(self, ymd):
|
||||
- date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
|
||||
- region_key = hmac_sha256(date_key, self.region.encode())
|
||||
- service_key = hmac_sha256(region_key, b's3')
|
||||
- signing_key = hmac_sha256(service_key, b'aws4_request')
|
||||
-
|
||||
- self.signing_key = (signing_key, ymd)
|
||||
-
|
||||
-
|
||||
-def hmac_sha256(key, msg, hex=False):
|
||||
- d = hmac.new(key, msg, hashlib.sha256)
|
||||
- if hex:
|
||||
- return d.hexdigest()
|
||||
- else:
|
||||
- return d.digest()
|
||||
diff --git a/src/s3ql/backends/s3c4.py b/src/s3ql/backends/s3c4.py
|
||||
new file mode 100644
|
||||
index 00000000..37ff0b7a
|
||||
--- /dev/null
|
||||
+++ b/src/s3ql/backends/s3c4.py
|
||||
@@ -0,0 +1,140 @@
|
||||
+'''
|
||||
+s3c4.py - this file is part of S3QL.
|
||||
+
|
||||
+Copyright © 2008 Nikolaus Rath <Nikolaus@rath.org>
|
||||
+
|
||||
+This work can be distributed under the terms of the GNU GPLv3.
|
||||
+'''
|
||||
+
|
||||
+import hashlib
|
||||
+import hmac
|
||||
+import logging
|
||||
+import re
|
||||
+import time
|
||||
+import urllib.parse
|
||||
+from xml.sax.saxutils import escape as xml_escape
|
||||
+
|
||||
+from ..logging import QuietError
|
||||
+from . import s3c
|
||||
+from .common import retry
|
||||
+from .s3c import get_S3Error
|
||||
+
|
||||
+log = logging.getLogger(__name__)
|
||||
+
|
||||
+# Maximum number of keys that can be deleted at once
|
||||
+MAX_KEYS = 1000
|
||||
+
|
||||
+# Pylint goes berserk with false positives
|
||||
+# pylint: disable=E1002,E1101
|
||||
+
|
||||
+
|
||||
+class Backend(s3c.Backend):
|
||||
+ """A backend to stored data in some S3 compatible storage service.
|
||||
+
|
||||
+ This classes uses AWS Signature V4 for authorization.
|
||||
+ """
|
||||
+
|
||||
+ known_options = s3c.Backend.known_options | {'sig-region'}
|
||||
+
|
||||
+ def __init__(self, options):
|
||||
+ self.sig_region = options.backend_options.get('sig-region', 'us-east-1')
|
||||
+ self.signing_key = None
|
||||
+ super().__init__(options)
|
||||
+
|
||||
+ def __str__(self):
|
||||
+ return 's3c4://%s/%s/%s' % (self.hostname, self.bucket_name, self.prefix)
|
||||
+
|
||||
+ def _authorize_request(self, method, path, headers, subres, query_string):
|
||||
+ '''Add authorization information to *headers*'''
|
||||
+
|
||||
+ # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
|
||||
+
|
||||
+ now = time.gmtime()
|
||||
+ # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
|
||||
+ # '%a, %d %b %Y %H:%M:%S GMT')
|
||||
+
|
||||
+ ymd = time.strftime('%Y%m%d', now)
|
||||
+ ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
|
||||
+
|
||||
+ # add non-standard port to host header, needed for correct signature
|
||||
+ if self.port != 443:
|
||||
+ headers['host'] = '%s:%s' % (self.hostname, self.port)
|
||||
+
|
||||
+ headers['x-amz-date'] = ymdhms
|
||||
+ headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
|
||||
+
|
||||
+ headers.pop('Authorization', None)
|
||||
+
|
||||
+ auth_strs = [method]
|
||||
+ auth_strs.append(urllib.parse.quote(path))
|
||||
+
|
||||
+ if query_string:
|
||||
+ s = urllib.parse.urlencode(
|
||||
+ query_string, doseq=True, quote_via=urllib.parse.quote
|
||||
+ ).split('&')
|
||||
+ else:
|
||||
+ s = []
|
||||
+ if subres:
|
||||
+ s.append(urllib.parse.quote(subres) + '=')
|
||||
+ if s:
|
||||
+ s = '&'.join(sorted(s))
|
||||
+ else:
|
||||
+ s = ''
|
||||
+ auth_strs.append(s)
|
||||
+
|
||||
+ # Headers
|
||||
+ sig_hdrs = sorted(x.lower() for x in headers.keys())
|
||||
+ for hdr in sig_hdrs:
|
||||
+ auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
|
||||
+ auth_strs.append('')
|
||||
+ auth_strs.append(';'.join(sig_hdrs))
|
||||
+ auth_strs.append(headers['x-amz-content-sha256'])
|
||||
+ can_req = '\n'.join(auth_strs)
|
||||
+ # log.debug('canonical request: %s', can_req)
|
||||
+
|
||||
+ can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
|
||||
+ str_to_sign = (
|
||||
+ "AWS4-HMAC-SHA256\n"
|
||||
+ + ymdhms
|
||||
+ + '\n'
|
||||
+ + '%s/%s/s3/aws4_request\n' % (ymd, self.sig_region)
|
||||
+ + can_req_hash
|
||||
+ )
|
||||
+ # log.debug('string to sign: %s', str_to_sign)
|
||||
+
|
||||
+ if self.signing_key is None or self.signing_key[1] != ymd:
|
||||
+ self.update_signing_key(ymd)
|
||||
+ signing_key = self.signing_key[0]
|
||||
+
|
||||
+ sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
|
||||
+
|
||||
+ cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
|
||||
+ self.login,
|
||||
+ now.tm_year,
|
||||
+ now.tm_mon,
|
||||
+ now.tm_mday,
|
||||
+ self.sig_region,
|
||||
+ )
|
||||
+
|
||||
+ headers['Authorization'] = (
|
||||
+ 'AWS4-HMAC-SHA256 '
|
||||
+ 'Credential=%s,'
|
||||
+ 'SignedHeaders=%s,'
|
||||
+ 'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
|
||||
+ )
|
||||
+
|
||||
+ def update_signing_key(self, ymd):
|
||||
+ date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
|
||||
+ region_key = hmac_sha256(date_key, self.sig_region.encode())
|
||||
+ service_key = hmac_sha256(region_key, b's3')
|
||||
+ signing_key = hmac_sha256(service_key, b'aws4_request')
|
||||
+
|
||||
+ self.signing_key = (signing_key, ymd)
|
||||
+
|
||||
+
|
||||
+def hmac_sha256(key, msg, hex=False):
|
||||
+ d = hmac.new(key, msg, hashlib.sha256)
|
||||
+ if hex:
|
||||
+ return d.hexdigest()
|
||||
+ else:
|
||||
+ return d.digest()
|
||||
diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py
|
||||
index 272e10c7..24ad50f4 100644
|
||||
--- a/src/s3ql/parse_args.py
|
||||
+++ b/src/s3ql/parse_args.py
|
||||
@@ -374,7 +374,7 @@ def storage_url_type(s):
|
||||
# slash (even when using a prefix), but we can't do that now because it
|
||||
# would make file systems created without trailing slash inaccessible.
|
||||
if re.match(r'^(s3|gs)://[^/]+$', s) or re.match(
|
||||
- r'^(s3c|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
|
||||
+ r'^(s3c|s3c4|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
|
||||
):
|
||||
s += '/'
|
||||
|
||||
diff --git a/tests/mock_server.py b/tests/mock_server.py
|
||||
index b453e705..e3084065 100644
|
||||
--- a/tests/mock_server.py
|
||||
+++ b/tests/mock_server.py
|
||||
@@ -292,6 +292,16 @@ def send_error(self, status, message=None, code='', resource='', extra_headers=N
|
||||
self.wfile.write(content)
|
||||
|
||||
|
||||
+class S3C4RequestHandler(S3CRequestHandler):
|
||||
+ '''Request Handler for s3c4 backend
|
||||
+
|
||||
+ Currently identical to S3CRequestHandler since mock request handlers
|
||||
+ do not check request signatures.
|
||||
+ '''
|
||||
+
|
||||
+ pass
|
||||
+
|
||||
+
|
||||
class BasicSwiftRequestHandler(S3CRequestHandler):
|
||||
'''A request handler implementing a subset of the OpenStack Swift Interface
|
||||
|
||||
@@ -569,6 +579,7 @@ def inline_error(http_status, body):
|
||||
#: corresponding storage urls
|
||||
handler_list = [
|
||||
(S3CRequestHandler, 's3c://%(host)s:%(port)d/s3ql_test'),
|
||||
+ (S3C4RequestHandler, 's3c4://%(host)s:%(port)d/s3ql_test'),
|
||||
# Special syntax only for testing against mock server
|
||||
(BasicSwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),
|
||||
(CopySwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),
|
BIN
secrets/keycloak-dbpass.age
Normal file
BIN
secrets/keycloak-dbpass.age
Normal file
Binary file not shown.
Loading…
Reference in a new issue