modules/ascensions/default.nix

@@ -0,0 +1,135 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  ascensionsDir = "/var/lib/ascensions";
+
+  ascensionType = { name, ... }: {
+    options = {
+      incantations = mkOption {
+        type = with types; functionTo (listOf package);
+        default = [];
+      };
+      distributed = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether to perform a distributed ascension using Consul KV.";
+      };
+      requiredBy = mkOption {
+        type = with types; listOf str;
+        default = [ "${name}.service" ];
+        description = "Services that require this ascension.";
+      };
+      before = mkOption {
+        type = with types; listOf str;
+        default = [];
+        description = "Run the ascension before these services.";
+      };
+      after = mkOption {
+        type = with types; listOf str;
+        default = [];
+        description = "Run the ascension after these services.";
+      };
+    };
+  };
+
+  builtinIncantations = with pkgs; rec {
+    execShellWith = extraPackages: script: writeShellScript "incantation" ''
+      export PATH='${makeBinPath ([ coreutils ] ++ extraPackages)}'
+      ${script}
+    '';
+
+    execShell = execShellWith [];
+
+    multiple = incantations: execShell (concatStringsSep " && " incantations);
+
+    move = from: to: execShell ''
+      test -e ${escapeShellArg from} && rm -rf ${escapeShellArg to}
+      mkdir -p "$(dirname ${escapeShellArg to})"
+      mv ${escapeShellArgs [ from to ]}
+    '';
+
+    chmod = mode: target: "chmod -R ${escapeShellArgs [ mode target ]}";
+  };
+
+  allIncantations = builtinIncantations // mapAttrs (_: mk: mk allIncantations) config.system.extraIncantations;
+
+  runIncantations = f: f allIncantations;
+
+  consul = config.services.consul.package;
+in
+
+{
+  options.system = {
+    ascensions = mkOption {
+      type = with types; attrsOf (submodule ascensionType);
+      default = {};
+    };
+    extraIncantations = mkOption {
+      type = with types; attrsOf (functionTo raw);
+      default = {};
+    };
+  };
+
+  config = {
+    systemd = {
+      tmpfiles.rules = [
+        "d ${ascensionsDir} 0755 root root - -"
+      ];
+
+      services = mapAttrs' (name: asc: {
+        name = "ascend-${name}";
+        value = let
+          incantations = runIncantations asc.incantations;
+          targetLevel = toString (length incantations);
+        in {
+          description = "Ascension for ${name}";
+          inherit (asc) requiredBy before;
+          after = asc.after ++ (lib.optional asc.distributed "consul.service");
+          serviceConfig.Type = "oneshot";
+          distributed.enable = asc.distributed;
+          script = ''
+            incantations=(${concatStringsSep " " incantations})
+            ${if asc.distributed then /*bash*/ ''
+              getLevel() {
+                ${consul}/bin/consul kv get 'ascensions/${name}/currentLevel'
+              }
+              setLevel() {
+                ${consul}/bin/consul kv put 'ascensions/${name}/currentLevel' "$1" >/dev/null
+              }
+              isEmpty() {
+                ! getLevel >/dev/null 2>/dev/null
+              }
+            '' else /*bash*/ ''
+              levelFile='${ascensionsDir}/${name}'
+              getLevel() {
+                echo "$(<"$levelFile")"
+              }
+              setLevel() {
+                echo "$1" > "$levelFile"
+              }
+              isEmpty() {
+                [[ ! -e "$levelFile" ]]
+              }
+            ''
+            }
+            if isEmpty; then
+              setLevel '${targetLevel}'
+              echo Initializing at level ${targetLevel}
+              exit 0
+            fi
+            cur=$(getLevel)
+            echo Current level: $cur
+            for lvl in $(seq $(($cur+1)) ${targetLevel}); do
+              echo Running incantation for level $lvl...
+              ''${incantations[$((lvl-1))]}
+              setLevel "$lvl"
+            done
+            echo All incantations complete, ascended to level $(getLevel)
+          '';
+        };
+      }) config.system.ascensions;
+    };
+  };
+}

modules/part.nix

@@ -7,6 +7,7 @@ in
 {
   flake.nixosModules = with config.flake.nixosModules; {
     autopatch = ./autopatch;
+    ascensions = ./ascensions;
     consul-distributed-services = ./consul-distributed-services;
     consul-service-registry = ./consul-service-registry;
     deploy-rs-receiver = ./deploy-rs-receiver;
@@ -45,6 +46,7 @@ in
 
     serverBase = group [
       machineBase
+      ascensions
       consul-distributed-services
       consul-service-registry
       deploy-rs-receiver

packages/checks/ascensions.nix

@@ -0,0 +1,98 @@
+{ testers, nixosModules }:
+
+let
+  dataDir = {
+    node1 = "/data/before";
+    node2 = "/data/nested/after";
+  };
+  kvPath = {
+    node1 = "before";
+    node2 = "after";
+  };
+in
+
+testers.runNixOSTest {
+  name = "ascensions";
+
+  imports = [
+    ./modules/consul.nix
+  ];
+
+  nodes = let
+    common = { config, lib, ... }: let
+      inherit (config.networking) hostName;
+    in {
+      imports = [
+        nixosModules.ascensions
+        nixosModules.systemd-extras
+        nixosModules.consul-distributed-services
+      ];
+      systemd.services = {
+        create-file = {
+          serviceConfig.Type = "oneshot";
+          script = ''
+            if ! test -e ${dataDir.${hostName}}/file.txt; then
+              mkdir -p ${dataDir.${hostName}}
+              echo "${hostName}" > ${dataDir.${hostName}}/file.txt
+            fi
+          '';
+        };
+        create-kv = {
+          serviceConfig.Type = "oneshot";
+          path = [ config.services.consul.package ];
+          script = ''
+            if ! consul kv get ${kvPath.${hostName}}; then
+              consul kv put ${kvPath.${hostName}} ${hostName}
+            fi
+          '';
+          environment.CONSUL_HTTP_ADDR = "consul:8500";
+        };
+        ascend-create-kv = {
+          environment.CONSUL_HTTP_ADDR = "consul:8500";
+        };
+      };
+      system.ascensions = {
+        create-file = {
+          before = [ "create-file.service" ];
+          incantations = m: with m; lib.optionals (hostName == "node2") [
+            (move dataDir.node1 "/data/somewhere/intermediate1")
+            (move "/data/somewhere/intermediate1" "/var/lib/intermediate2")
+            (move "/var/lib/intermediate2" dataDir.node2)
+          ];
+        };
+        create-kv = {
+          distributed = true;
+          before = [ "create-kv.service" ];
+          incantations = m: with m; lib.optionals (hostName == "node2") [
+            (execShellWith [ config.services.consul.package ] ''
+              consul kv put intermediate/data $(consul kv get ${kvPath.node1})
+              consul kv delete ${kvPath.node1}
+            '')
+            (execShellWith [ config.services.consul.package ] ''
+              consul kv put ${kvPath.node2} $(consul kv get intermediate/data)
+              consul kv delete intermediate/data
+            '')
+          ];
+        };
+      };
+    };
+  in {
+    node1.imports = [ common ];
+    node2.imports = [ common ];
+  };
+  testScript = /*python*/ ''
+    start_all()
+    consul.wait_for_unit("consul.service")
+    consul.wait_until_succeeds("CONSUL_HTTP_ADDR=consul:8500 consul members")
+    node1.wait_for_unit("multi-user.target")
+    node1.succeed("systemctl start create-file create-kv")
+    node1.succeed("tar cvf /tmp/shared/data.tar /data /var/lib/ascensions")
+
+    node2.wait_for_unit("multi-user.target")
+    node2.succeed("tar xvf /tmp/shared/data.tar -C /")
+    node2.succeed("systemctl start create-file create-kv")
+
+    assert "node1" in node2.succeed("cat ${dataDir.node2}/file.txt")
+    assert "node1" in consul.succeed("CONSUL_HTTP_ADDR=consul:8500 consul kv get ${kvPath.node2}")
+  '';
+}

packages/checks/default.nix

@@ -1,8 +1,12 @@
-{ config, self, ... }:
+{ config, lib, self, ... }:
 
 {
-  perSystem = { filters, pkgs, self', ... }: {
+  perSystem = { filters, pkgs, self', system, ... }: {
-    checks = filters.doFilter filters.checks {
+    checks = lib.mkIf (system == "x86_64-linux") {
+      ascensions = pkgs.callPackage ./ascensions.nix {
+        inherit (self) nixosModules;
+      };
+
       jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix {
         inherit (self'.packages) jellyfin;
         inherit (config) cluster;

packages/checks/modules/consul.nix

@@ -0,0 +1,19 @@
+{ config, ... }:
+
+{
+  extraBaseModules = {
+    services.consul.extraConfig.addresses.http = config.nodes.consul.networking.primaryIPAddress;
+  };
+
+  nodes.consul = { config, ... }: {
+    networking.firewall.allowedTCPPorts = [ 8500 ];
+    services.consul = {
+      enable = true;
+      extraConfig = {
+        bind_addr = config.networking.primaryIPAddress;
+        server = true;
+        bootstrap_expect = 1;
+      };
+    };
+  };
+}

packages/system-filter.nix

@@ -11,11 +11,4 @@
     searxng = [ "x86_64-linux" ];
     tempo = [ "x86_64-linux" ];
   };
-  checks = {
-    jellyfin-stateless = [ "x86_64-linux" ];
-    keycloak = [ "x86_64-linux" ];
-    patroni = [ "x86_64-linux" ];
-    searxng = [ "x86_64-linux" ];
-    tempo = [ "x86_64-linux" ];
-  };
 }