Compare commits

...

3 commits

Author SHA1 Message Date
Max
727af63d6f cluster/services/attic: prepare for standalone ascension 2023-08-29 01:16:37 +02:00
Max
0f3bd138e5 checks/ascensions: init 2023-08-29 00:55:34 +02:00
Max
a8a564a2bb modules/ascension: init 2023-08-29 00:54:42 +02:00
6 changed files with 251 additions and 0 deletions

View file

@ -9,6 +9,12 @@ in
depot.inputs.attic.nixosModules.atticd depot.inputs.attic.nixosModules.atticd
]; ];
ascensions.attic-standalone = {
requiredBy = [ "attic.service" ];
before = [ "attic.service" ];
incantations = i: [ ];
};
age.secrets.atticServerToken.file = ./attic-server-token.age; age.secrets.atticServerToken.file = ./attic-server-token.age;
links.atticServer.protocol = "http"; links.atticServer.protocol = "http";

View file

@ -0,0 +1,125 @@
{ config, lib, pkgs, ... }:
with lib;
let
ascensionsDir = "/var/lib/ascensions";
ascensionType = { name, ... }: {
options = {
incantations = mkOption {
type = with types; functionTo (listOf package);
default = [];
};
distributed = mkOption {
type = types.bool;
default = false;
description = "Whether to perform a distributed ascension using Consul KV.";
};
requiredBy = mkOption {
type = with types; listOf str;
default = [ "${name}.service" ];
description = "Services that require this ascension.";
};
before = mkOption {
type = with types; listOf str;
default = [];
description = "Run the ascension before these services.";
};
after = mkOption {
type = with types; listOf str;
default = [];
description = "Run the ascension after these services.";
};
};
};
runIncantations = f: with pkgs; f rec {
execShellWith = extraPackages: script: writeShellScript "incantation" ''
export PATH='${makeBinPath ([ coreutils ] ++ extraPackages)}'
${script}
'';
execShell = execShellWith [];
multiple = incantations: execShell (concatStringsSep " && " incantations);
move = from: to: execShell ''
test -e ${escapeShellArg from} && rm -rf ${escapeShellArg to}
mkdir -p "$(dirname ${escapeShellArg to})"
mv ${escapeShellArgs [ from to ]}
'';
chmod = mode: target: "chmod -R ${escapeShellArgs [ mode target ]}";
};
consul = config.services.consul.package;
in
{
options.system.ascensions = mkOption {
type = with types; attrsOf (submodule ascensionType);
default = {};
};
config = {
systemd = {
tmpfiles.rules = [
"d ${ascensionsDir} 0755 root root - -"
];
services = mapAttrs' (name: asc: {
name = "ascend-${name}";
value = let
incantations = runIncantations asc.incantations;
targetLevel = toString (length incantations);
in {
description = "Ascension for ${name}";
inherit (asc) requiredBy before;
after = asc.after ++ (lib.optional asc.distributed "consul.service");
serviceConfig.Type = "oneshot";
distributed.enable = asc.distributed;
script = ''
incantations=(${concatStringsSep " " incantations})
${if asc.distributed then /*bash*/ ''
getLevel() {
${consul}/bin/consul kv get 'ascensions/${name}/currentLevel'
}
setLevel() {
${consul}/bin/consul kv put 'ascensions/${name}/currentLevel' "$1" >/dev/null
}
isEmpty() {
! getLevel >/dev/null 2>/dev/null
}
'' else /*bash*/ ''
levelFile='${ascensionsDir}/${name}'
getLevel() {
echo "$(<"$levelFile")"
}
setLevel() {
echo "$1" > "$levelFile"
}
isEmpty() {
[[ ! -e "$levelFile" ]]
}
''
}
if isEmpty; then
setLevel '${targetLevel}'
echo Initializing at level ${targetLevel}
exit 0
fi
cur=$(getLevel)
echo Current level: $cur
for lvl in $(seq $(($cur+1)) ${targetLevel}); do
echo Running incantation for level $lvl...
''${incantations[$((lvl-1))]}
setLevel "$lvl"
done
echo All incantations complete, ascended to level $(getLevel)
'';
};
}) config.system.ascensions;
};
};
}

View file

@ -7,6 +7,7 @@ in
{ {
flake.nixosModules = with config.flake.nixosModules; { flake.nixosModules = with config.flake.nixosModules; {
autopatch = ./autopatch; autopatch = ./autopatch;
ascensions = ./ascensions;
consul-distributed-services = ./consul-distributed-services; consul-distributed-services = ./consul-distributed-services;
consul-service-registry = ./consul-service-registry; consul-service-registry = ./consul-service-registry;
deploy-rs-receiver = ./deploy-rs-receiver; deploy-rs-receiver = ./deploy-rs-receiver;
@ -45,6 +46,7 @@ in
serverBase = group [ serverBase = group [
machineBase machineBase
ascensions
consul-distributed-services consul-distributed-services
consul-service-registry consul-service-registry
deploy-rs-receiver deploy-rs-receiver

View file

@ -0,0 +1,113 @@
{ nixosTest, nixosModules }:
let
dataDir = {
node1 = "/data/before";
node2 = "/data/nested/after";
};
kvPath = {
node1 = "before";
node2 = "after";
};
addr = {
consul = "10.0.0.10";
node1 = "10.0.0.1";
node2 = "10.0.0.2";
};
in
nixosTest {
name = "ascensions";
nodes = let
network = { config, lib, ... }: {
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = addr.${config.networking.hostName};
prefixLength = 24;
}
];
};
common = { config, lib, ... }: let
inherit (config.networking) hostName;
in {
imports = [
nixosModules.ascensions
nixosModules.systemd-extras
nixosModules.consul-distributed-services
];
systemd.services = {
create-file = {
serviceConfig.Type = "oneshot";
script = ''
if ! test -e ${dataDir.${hostName}}/file.txt; then
mkdir -p ${dataDir.${hostName}}
echo "${hostName}" > ${dataDir.${hostName}}/file.txt
fi
'';
};
create-kv = {
serviceConfig.Type = "oneshot";
path = [ config.services.consul.package ];
script = ''
if ! consul kv get ${kvPath.${hostName}}; then
consul kv put ${kvPath.${hostName}} ${hostName}
fi
'';
environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
};
ascend-create-kv = {
environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
};
};
system.ascensions = {
create-file = {
before = [ "create-file.service" ];
incantations = m: with m; lib.optionals (hostName == "node2") [
(move dataDir.node1 dataDir.node2)
];
};
create-kv = {
distributed = true;
before = [ "create-kv.service" ];
incantations = m: with m; lib.optionals (hostName == "node2") [
(execShellWith [ config.services.consul.package ] ''
consul kv put ${kvPath.node2} $(consul kv get ${kvPath.node1})
consul kv delete ${kvPath.node1}
'')
];
};
};
};
in {
consul = {
imports = [ network ];
networking.firewall.allowedTCPPorts = [ 8500 ];
services.consul = {
enable = true;
extraConfig = {
addresses.http = addr.consul;
bind_addr = addr.consul;
server = true;
bootstrap_expect = 1;
};
};
};
node1.imports = [ network common ];
node2.imports = [ network common ];
};
testScript = /*python*/ ''
start_all()
consul.wait_for_unit("consul.service")
consul.wait_until_succeeds("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul members")
node1.wait_for_unit("multi-user.target")
node1.succeed("systemctl start create-file create-kv")
node1.succeed("tar cvf /tmp/shared/data.tar /data /var/lib/ascensions")
node2.wait_for_unit("multi-user.target")
node2.succeed("tar xvf /tmp/shared/data.tar -C /")
node2.succeed("systemctl start create-file create-kv")
assert "node1" in node2.succeed("cat ${dataDir.node2}/file.txt")
assert "node1" in consul.succeed("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul kv get ${kvPath.node2}")
'';
}

View file

@ -9,6 +9,10 @@
}; };
in { in {
checks = filters.doFilter filters.checks { checks = filters.doFilter filters.checks {
ascensions = pkgs.callPackage ./ascensions.nix {
inherit (self) nixosModules;
};
jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix { jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix {
inherit (self'.packages) jellyfin; inherit (self'.packages) jellyfin;
inherit fakeCluster; inherit fakeCluster;

View file

@ -12,6 +12,7 @@
tempo = [ "x86_64-linux" ]; tempo = [ "x86_64-linux" ];
}; };
checks = { checks = {
ascensions = [ "x86_64-linux" ];
jellyfin-stateless = [ "x86_64-linux" ]; jellyfin-stateless = [ "x86_64-linux" ];
keycloak = [ "x86_64-linux" ]; keycloak = [ "x86_64-linux" ];
patroni = [ "x86_64-linux" ]; patroni = [ "x86_64-linux" ];