cluster/services/attic: prepare for standalone ascension

cluster/services/attic/server.nix

@@ -9,6 +9,12 @@ in
     depot.inputs.attic.nixosModules.atticd
   ];
 
+  ascensions.attic-standalone = {
+    requiredBy = [ "attic.service" ];
+    before = [ "attic.service" ];
+    incantations = i: [ ];
+  };
+
   age.secrets.atticServerToken.file = ./attic-server-token.age;
 
   links.atticServer.protocol = "http";

modules/ascensions/default.nix

@@ -0,0 +1,125 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  ascensionsDir = "/var/lib/ascensions";
+
+  ascensionType = { name, ... }: {
+    options = {
+      incantations = mkOption {
+        type = with types; functionTo (listOf package);
+        default = [];
+      };
+      distributed = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether to perform a distributed ascension using Consul KV.";
+      };
+      requiredBy = mkOption {
+        type = with types; listOf str;
+        default = [ "${name}.service" ];
+        description = "Services that require this ascension.";
+      };
+      before = mkOption {
+        type = with types; listOf str;
+        default = [];
+        description = "Run the ascension before these services.";
+      };
+      after = mkOption {
+        type = with types; listOf str;
+        default = [];
+        description = "Run the ascension after these services.";
+      };
+    };
+  };
+
+  runIncantations = f: with pkgs; f rec {
+    execShellWith = extraPackages: script: writeShellScript "incantation" ''
+      export PATH='${makeBinPath ([ coreutils ] ++ extraPackages)}'
+      ${script}
+    '';
+
+    execShell = execShellWith [];
+
+    multiple = incantations: execShell (concatStringsSep " && " incantations);
+
+    move = from: to: execShell ''
+      test -e ${escapeShellArg from} && rm -rf ${escapeShellArg to}
+      mkdir -p "$(dirname ${escapeShellArg to})"
+      mv ${escapeShellArgs [ from to ]}
+    '';
+
+    chmod = mode: target: "chmod -R ${escapeShellArgs [ mode target ]}";
+  };
+
+  consul = config.services.consul.package;
+in
+
+{
+  options.system.ascensions = mkOption {
+    type = with types; attrsOf (submodule ascensionType);
+    default = {};
+  };
+
+  config = {
+    systemd = {
+      tmpfiles.rules = [
+        "d ${ascensionsDir} 0755 root root - -"
+      ];
+
+      services = mapAttrs' (name: asc: {
+        name = "ascend-${name}";
+        value = let
+          incantations = runIncantations asc.incantations;
+          targetLevel = toString (length incantations);
+        in {
+          description = "Ascension for ${name}";
+          inherit (asc) requiredBy before;
+          after = asc.after ++ (lib.optional asc.distributed "consul.service");
+          serviceConfig.Type = "oneshot";
+          distributed.enable = asc.distributed;
+          script = ''
+            incantations=(${concatStringsSep " " incantations})
+            ${if asc.distributed then /*bash*/ ''
+              getLevel() {
+                ${consul}/bin/consul kv get 'ascensions/${name}/currentLevel'
+              }
+              setLevel() {
+                ${consul}/bin/consul kv put 'ascensions/${name}/currentLevel' "$1" >/dev/null
+              }
+              isEmpty() {
+                ! getLevel >/dev/null 2>/dev/null
+              }
+            '' else /*bash*/ ''
+              levelFile='${ascensionsDir}/${name}'
+              getLevel() {
+                echo "$(<"$levelFile")"
+              }
+              setLevel() {
+                echo "$1" > "$levelFile"
+              }
+              isEmpty() {
+                [[ ! -e "$levelFile" ]]
+              }
+            ''
+            }
+            if isEmpty; then
+              setLevel '${targetLevel}'
+              echo Initializing at level ${targetLevel}
+              exit 0
+            fi
+            cur=$(getLevel)
+            echo Current level: $cur
+            for lvl in $(seq $(($cur+1)) ${targetLevel}); do
+              echo Running incantation for level $lvl...
+              ''${incantations[$((lvl-1))]}
+              setLevel "$lvl"
+            done
+            echo All incantations complete, ascended to level $(getLevel)
+          '';
+        };
+      }) config.system.ascensions;
+    };
+  };
+}

modules/part.nix

@@ -7,6 +7,7 @@ in
 {
   flake.nixosModules = with config.flake.nixosModules; {
     autopatch = ./autopatch;
+    ascensions = ./ascensions;
     consul-distributed-services = ./consul-distributed-services;
     consul-service-registry = ./consul-service-registry;
     deploy-rs-receiver = ./deploy-rs-receiver;
@@ -45,6 +46,7 @@ in
 
     serverBase = group [
       machineBase
+      ascensions
       consul-distributed-services
       consul-service-registry
       deploy-rs-receiver

packages/checks/ascensions.nix

@@ -0,0 +1,113 @@
+{ nixosTest, nixosModules }:
+
+let
+  dataDir = {
+    node1 = "/data/before";
+    node2 = "/data/nested/after";
+  };
+  kvPath = {
+    node1 = "before";
+    node2 = "after";
+  };
+  addr = {
+    consul = "10.0.0.10";
+    node1 = "10.0.0.1";
+    node2 = "10.0.0.2";
+  };
+in
+
+nixosTest {
+  name = "ascensions";
+  nodes = let
+    network = { config, lib, ... }: {
+      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
+        {
+          address = addr.${config.networking.hostName};
+          prefixLength = 24;
+        }
+      ];
+    };
+    common = { config, lib, ... }: let
+      inherit (config.networking) hostName;
+    in {
+      imports = [
+        nixosModules.ascensions
+        nixosModules.systemd-extras
+        nixosModules.consul-distributed-services
+      ];
+      systemd.services = {
+        create-file = {
+          serviceConfig.Type = "oneshot";
+          script = ''
+            if ! test -e ${dataDir.${hostName}}/file.txt; then
+              mkdir -p ${dataDir.${hostName}}
+              echo "${hostName}" > ${dataDir.${hostName}}/file.txt
+            fi
+          '';
+        };
+        create-kv = {
+          serviceConfig.Type = "oneshot";
+          path = [ config.services.consul.package ];
+          script = ''
+            if ! consul kv get ${kvPath.${hostName}}; then
+              consul kv put ${kvPath.${hostName}} ${hostName}
+            fi
+          '';
+          environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
+        };
+        ascend-create-kv = {
+          environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
+        };
+      };
+      system.ascensions = {
+        create-file = {
+          before = [ "create-file.service" ];
+          incantations = m: with m; lib.optionals (hostName == "node2") [
+            (move dataDir.node1 dataDir.node2)
+          ];
+        };
+        create-kv = {
+          distributed = true;
+          before = [ "create-kv.service" ];
+          incantations = m: with m; lib.optionals (hostName == "node2") [
+            (execShellWith [ config.services.consul.package ] ''
+              consul kv put ${kvPath.node2} $(consul kv get ${kvPath.node1})
+              consul kv delete ${kvPath.node1}
+            '')
+          ];
+        };
+      };
+    };
+  in {
+    consul = {
+      imports = [ network ];
+      networking.firewall.allowedTCPPorts = [ 8500 ];
+      services.consul = {
+        enable = true;
+        extraConfig = {
+          addresses.http = addr.consul;
+          bind_addr = addr.consul;
+          server = true;
+          bootstrap_expect = 1;
+        };
+      };
+    };
+    node1.imports = [ network common ];
+    node2.imports = [ network common ];
+  };
+  testScript = /*python*/ ''
+    start_all()
+    consul.wait_for_unit("consul.service")
+    consul.wait_until_succeeds("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul members")
+    node1.wait_for_unit("multi-user.target")
+    node1.succeed("systemctl start create-file create-kv")
+    node1.succeed("tar cvf /tmp/shared/data.tar /data /var/lib/ascensions")
+
+    node2.wait_for_unit("multi-user.target")
+    node2.succeed("tar xvf /tmp/shared/data.tar -C /")
+    node2.succeed("systemctl start create-file create-kv")
+
+    assert "node1" in node2.succeed("cat ${dataDir.node2}/file.txt")
+    assert "node1" in consul.succeed("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul kv get ${kvPath.node2}")
+  '';
+}

packages/checks/default.nix

@@ -9,6 +9,10 @@
     };
   in {
     checks = filters.doFilter filters.checks {
+      ascensions = pkgs.callPackage ./ascensions.nix {
+        inherit (self) nixosModules;
+      };
+
       jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix {
         inherit (self'.packages) jellyfin;
         inherit fakeCluster;

packages/system-filter.nix

@@ -12,6 +12,7 @@
     tempo = [ "x86_64-linux" ];
   };
   checks = {
+    ascensions = [ "x86_64-linux" ];
     jellyfin-stateless = [ "x86_64-linux" ];
     keycloak = [ "x86_64-linux" ];
     patroni = [ "x86_64-linux" ];