From f9daec023f27dcf666909201b9bdd713afb7310f Mon Sep 17 00:00:00 2001
From: Max <max@privatevoid.net>
Date: Sun, 27 Aug 2023 22:52:08 +0200
Subject: [PATCH 1/6] modules/ascension: init

---
 modules/ascensions/default.nix | 125 +++++++++++++++++++++++++++++++++
 modules/part.nix               |   2 +
 2 files changed, 127 insertions(+)
 create mode 100644 modules/ascensions/default.nix

diff --git a/modules/ascensions/default.nix b/modules/ascensions/default.nix
new file mode 100644
index 0000000..4fe9dd8
--- /dev/null
+++ b/modules/ascensions/default.nix
@@ -0,0 +1,125 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  ascensionsDir = "/var/lib/ascensions";
+
+  ascensionType = { name, ... }: {
+    options = {
+      incantations = mkOption {
+        type = with types; functionTo (listOf package);
+        default = [];
+      };
+      distributed = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether to perform a distributed ascension using Consul KV.";
+      };
+      requiredBy = mkOption {
+        type = with types; listOf str;
+        default = [ "${name}.service" ];
+        description = "Services that require this ascension.";
+      };
+      before = mkOption {
+        type = with types; listOf str;
+        default = [];
+        description = "Run the ascension before these services.";
+      };
+      after = mkOption {
+        type = with types; listOf str;
+        default = [];
+        description = "Run the ascension after these services.";
+      };
+    };
+  };
+
+  runIncantations = f: with pkgs; f rec {
+    execShellWith = extraPackages: script: writeShellScript "incantation" ''
+      export PATH='${makeBinPath ([ coreutils ] ++ extraPackages)}'
+      ${script}
+    '';
+
+    execShell = execShellWith [];
+
+    multiple = incantations: execShell (concatStringsSep " && " incantations);
+
+    move = from: to: execShell ''
+      test -e ${escapeShellArg from} && rm -rf ${escapeShellArg to}
+      mkdir -p "$(dirname ${escapeShellArg to})"
+      mv ${escapeShellArgs [ from to ]}
+    '';
+
+    chmod = mode: target: "chmod -R ${escapeShellArgs [ mode target ]}";
+  };
+
+  consul = config.services.consul.package;
+in
+
+{
+  options.system.ascensions = mkOption {
+    type = with types; attrsOf (submodule ascensionType);
+    default = {};
+  };
+
+  config = {
+    systemd = {
+      tmpfiles.rules = [
+        "d ${ascensionsDir} 0755 root root - -"
+      ];
+
+      services = mapAttrs' (name: asc: {
+        name = "ascend-${name}";
+        value = let
+          incantations = runIncantations asc.incantations;
+          targetLevel = toString (length incantations);
+        in {
+          description = "Ascension for ${name}";
+          inherit (asc) requiredBy before;
+          after = asc.after ++ (lib.optional asc.distributed "consul.service");
+          serviceConfig.Type = "oneshot";
+          distributed.enable = asc.distributed;
+          script = ''
+            incantations=(${concatStringsSep " " incantations})
+            ${if asc.distributed then /*bash*/ ''
+              getLevel() {
+                ${consul}/bin/consul kv get 'ascensions/${name}/currentLevel'
+              }
+              setLevel() {
+                ${consul}/bin/consul kv put 'ascensions/${name}/currentLevel' "$1" >/dev/null
+              }
+              isEmpty() {
+                ! getLevel >/dev/null 2>/dev/null
+              }
+            '' else /*bash*/ ''
+              levelFile='${ascensionsDir}/${name}'
+              getLevel() {
+                echo "$(<"$levelFile")"
+              }
+              setLevel() {
+                echo "$1" > "$levelFile"
+              }
+              isEmpty() {
+                [[ ! -e "$levelFile" ]]
+              }
+            ''
+            }
+            if isEmpty; then
+              setLevel '${targetLevel}'
+              echo Initializing at level ${targetLevel}
+              exit 0
+            fi
+            cur=$(getLevel)
+            echo Current level: $cur
+            for lvl in $(seq $(($cur+1)) ${targetLevel}); do
+              echo Running incantation for level $lvl...
+              ''${incantations[$((lvl-1))]}
+              setLevel "$lvl"
+            done
+            echo All incantations complete, ascended to level $(getLevel)
+          '';
+        };
+      }) config.system.ascensions;
+    };
+  };
+}
diff --git a/modules/part.nix b/modules/part.nix
index 00159ad..7546353 100644
--- a/modules/part.nix
+++ b/modules/part.nix
@@ -7,6 +7,7 @@ in
 {
   flake.nixosModules = with config.flake.nixosModules; {
     autopatch = ./autopatch;
+    ascensions = ./ascensions;
     consul-distributed-services = ./consul-distributed-services;
     consul-service-registry = ./consul-service-registry;
     deploy-rs-receiver = ./deploy-rs-receiver;
@@ -45,6 +46,7 @@ in
 
     serverBase = group [
       machineBase
+      ascensions
       consul-distributed-services
       consul-service-registry
       deploy-rs-receiver

From 48d635db6a6ac1047619bb6cc5fabd04a90e428a Mon Sep 17 00:00:00 2001
From: Max <max@privatevoid.net>
Date: Tue, 29 Aug 2023 00:55:34 +0200
Subject: [PATCH 2/6] checks/ascensions: init

---
 packages/checks/ascensions.nix | 113 +++++++++++++++++++++++++++++++++
 packages/checks/default.nix    |   4 ++
 packages/system-filter.nix     |   1 +
 3 files changed, 118 insertions(+)
 create mode 100644 packages/checks/ascensions.nix

diff --git a/packages/checks/ascensions.nix b/packages/checks/ascensions.nix
new file mode 100644
index 0000000..6e04906
--- /dev/null
+++ b/packages/checks/ascensions.nix
@@ -0,0 +1,113 @@
+{ nixosTest, nixosModules }:
+
+let
+  dataDir = {
+    node1 = "/data/before";
+    node2 = "/data/nested/after";
+  };
+  kvPath = {
+    node1 = "before";
+    node2 = "after";
+  };
+  addr = {
+    consul = "10.0.0.10";
+    node1 = "10.0.0.1";
+    node2 = "10.0.0.2";
+  };
+in
+
+nixosTest {
+  name = "ascensions";
+  nodes = let
+    network = { config, lib, ... }: {
+      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
+        {
+          address = addr.${config.networking.hostName};
+          prefixLength = 24;
+        }
+      ];
+    };
+    common = { config, lib, ... }: let
+      inherit (config.networking) hostName;
+    in {
+      imports = [
+        nixosModules.ascensions
+        nixosModules.systemd-extras
+        nixosModules.consul-distributed-services
+      ];
+      systemd.services = {
+        create-file = {
+          serviceConfig.Type = "oneshot";
+          script = ''
+            if ! test -e ${dataDir.${hostName}}/file.txt; then
+              mkdir -p ${dataDir.${hostName}}
+              echo "${hostName}" > ${dataDir.${hostName}}/file.txt
+            fi
+          '';
+        };
+        create-kv = {
+          serviceConfig.Type = "oneshot";
+          path = [ config.services.consul.package ];
+          script = ''
+            if ! consul kv get ${kvPath.${hostName}}; then
+              consul kv put ${kvPath.${hostName}} ${hostName}
+            fi
+          '';
+          environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
+        };
+        ascend-create-kv = {
+          environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
+        };
+      };
+      system.ascensions = {
+        create-file = {
+          before = [ "create-file.service" ];
+          incantations = m: with m; lib.optionals (hostName == "node2") [
+            (move dataDir.node1 dataDir.node2)
+          ];
+        };
+        create-kv = {
+          distributed = true;
+          before = [ "create-kv.service" ];
+          incantations = m: with m; lib.optionals (hostName == "node2") [
+            (execShellWith [ config.services.consul.package ] ''
+              consul kv put ${kvPath.node2} $(consul kv get ${kvPath.node1})
+              consul kv delete ${kvPath.node1}
+            '')
+          ];
+        };
+      };
+    };
+  in {
+    consul = {
+      imports = [ network ];
+      networking.firewall.allowedTCPPorts = [ 8500 ];
+      services.consul = {
+        enable = true;
+        extraConfig = {
+          addresses.http = addr.consul;
+          bind_addr = addr.consul;
+          server = true;
+          bootstrap_expect = 1;
+        };
+      };
+    };
+    node1.imports = [ network common ];
+    node2.imports = [ network common ];
+  };
+  testScript = /*python*/ ''
+    start_all()
+    consul.wait_for_unit("consul.service")
+    consul.wait_until_succeeds("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul members")
+    node1.wait_for_unit("multi-user.target")
+    node1.succeed("systemctl start create-file create-kv")
+    node1.succeed("tar cvf /tmp/shared/data.tar /data /var/lib/ascensions")
+
+    node2.wait_for_unit("multi-user.target")
+    node2.succeed("tar xvf /tmp/shared/data.tar -C /")
+    node2.succeed("systemctl start create-file create-kv")
+
+    assert "node1" in node2.succeed("cat ${dataDir.node2}/file.txt")
+    assert "node1" in consul.succeed("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul kv get ${kvPath.node2}")
+  '';
+}
diff --git a/packages/checks/default.nix b/packages/checks/default.nix
index b51b4c7..ffb02d6 100644
--- a/packages/checks/default.nix
+++ b/packages/checks/default.nix
@@ -3,6 +3,10 @@
 {
   perSystem = { filters, pkgs, self', ... }: {
     checks = filters.doFilter filters.checks {
+      ascensions = pkgs.callPackage ./ascensions.nix {
+        inherit (self) nixosModules;
+      };
+
       jellyfin-stateless = pkgs.callPackage ./jellyfin-stateless.nix {
         inherit (self'.packages) jellyfin;
         inherit (config) cluster;
diff --git a/packages/system-filter.nix b/packages/system-filter.nix
index f2f0f82..f66cffb 100644
--- a/packages/system-filter.nix
+++ b/packages/system-filter.nix
@@ -12,6 +12,7 @@
     tempo = [ "x86_64-linux" ];
   };
   checks = {
+    ascensions = [ "x86_64-linux" ];
     jellyfin-stateless = [ "x86_64-linux" ];
     keycloak = [ "x86_64-linux" ];
     patroni = [ "x86_64-linux" ];

From cfa64284faa257e6a710fa22d393071ae853a2df Mon Sep 17 00:00:00 2001
From: Max <max@privatevoid.net>
Date: Sun, 3 Sep 2023 03:15:45 +0200
Subject: [PATCH 3/6] checks: add generic consul module

---
 packages/checks/ascensions.nix     | 47 +++++++++---------------------
 packages/checks/modules/consul.nix | 19 ++++++++++++
 2 files changed, 32 insertions(+), 34 deletions(-)
 create mode 100644 packages/checks/modules/consul.nix

diff --git a/packages/checks/ascensions.nix b/packages/checks/ascensions.nix
index 6e04906..8efb819 100644
--- a/packages/checks/ascensions.nix
+++ b/packages/checks/ascensions.nix
@@ -1,4 +1,4 @@
-{ nixosTest, nixosModules }:
+{ testers, nixosModules }:
 
 let
   dataDir = {
@@ -9,24 +9,16 @@ let
     node1 = "before";
     node2 = "after";
   };
-  addr = {
-    consul = "10.0.0.10";
-    node1 = "10.0.0.1";
-    node2 = "10.0.0.2";
-  };
 in
 
-nixosTest {
+testers.runNixOSTest {
   name = "ascensions";
+
+  imports = [
+    ./modules/consul.nix
+  ];
+
   nodes = let
-    network = { config, lib, ... }: {
-      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
-        {
-          address = addr.${config.networking.hostName};
-          prefixLength = 24;
-        }
-      ];
-    };
     common = { config, lib, ... }: let
       inherit (config.networking) hostName;
     in {
@@ -53,10 +45,10 @@ nixosTest {
               consul kv put ${kvPath.${hostName}} ${hostName}
             fi
           '';
-          environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
+          environment.CONSUL_HTTP_ADDR = "consul:8500";
         };
         ascend-create-kv = {
-          environment.CONSUL_HTTP_ADDR = "${addr.consul}:8500";
+          environment.CONSUL_HTTP_ADDR = "consul:8500";
         };
       };
       system.ascensions = {
@@ -79,26 +71,13 @@ nixosTest {
       };
     };
   in {
-    consul = {
-      imports = [ network ];
-      networking.firewall.allowedTCPPorts = [ 8500 ];
-      services.consul = {
-        enable = true;
-        extraConfig = {
-          addresses.http = addr.consul;
-          bind_addr = addr.consul;
-          server = true;
-          bootstrap_expect = 1;
-        };
-      };
-    };
-    node1.imports = [ network common ];
-    node2.imports = [ network common ];
+    node1.imports = [ common ];
+    node2.imports = [ common ];
   };
   testScript = /*python*/ ''
     start_all()
     consul.wait_for_unit("consul.service")
-    consul.wait_until_succeeds("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul members")
+    consul.wait_until_succeeds("CONSUL_HTTP_ADDR=consul:8500 consul members")
     node1.wait_for_unit("multi-user.target")
     node1.succeed("systemctl start create-file create-kv")
     node1.succeed("tar cvf /tmp/shared/data.tar /data /var/lib/ascensions")
@@ -108,6 +87,6 @@ nixosTest {
     node2.succeed("systemctl start create-file create-kv")
 
     assert "node1" in node2.succeed("cat ${dataDir.node2}/file.txt")
-    assert "node1" in consul.succeed("CONSUL_HTTP_ADDR=${addr.consul}:8500 consul kv get ${kvPath.node2}")
+    assert "node1" in consul.succeed("CONSUL_HTTP_ADDR=consul:8500 consul kv get ${kvPath.node2}")
   '';
 }
diff --git a/packages/checks/modules/consul.nix b/packages/checks/modules/consul.nix
new file mode 100644
index 0000000..8dbb599
--- /dev/null
+++ b/packages/checks/modules/consul.nix
@@ -0,0 +1,19 @@
+{ config, ... }:
+
+{
+  extraBaseModules = {
+    services.consul.extraConfig.addresses.http = config.nodes.consul.networking.primaryIPAddress;
+  };
+
+  nodes.consul = { config, ... }: {
+    networking.firewall.allowedTCPPorts = [ 8500 ];
+    services.consul = {
+      enable = true;
+      extraConfig = {
+        bind_addr = config.networking.primaryIPAddress;
+        server = true;
+        bootstrap_expect = 1;
+      };
+    };
+  };
+}

From 72f3cead67a1dc0b3b6e698bdccee41bbfbe7765 Mon Sep 17 00:00:00 2001
From: Max <max@privatevoid.net>
Date: Sun, 3 Sep 2023 03:21:43 +0200
Subject: [PATCH 4/6] checks/ascensions: add more incantations

---
 packages/checks/ascensions.nix | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/checks/ascensions.nix b/packages/checks/ascensions.nix
index 8efb819..c601790 100644
--- a/packages/checks/ascensions.nix
+++ b/packages/checks/ascensions.nix
@@ -55,7 +55,9 @@ testers.runNixOSTest {
         create-file = {
           before = [ "create-file.service" ];
           incantations = m: with m; lib.optionals (hostName == "node2") [
-            (move dataDir.node1 dataDir.node2)
+            (move dataDir.node1 "/data/somewhere/intermediate1")
+            (move "/data/somewhere/intermediate1" "/var/lib/intermediate2")
+            (move "/var/lib/intermediate2" dataDir.node2)
           ];
         };
         create-kv = {
@@ -63,9 +65,13 @@ testers.runNixOSTest {
           before = [ "create-kv.service" ];
           incantations = m: with m; lib.optionals (hostName == "node2") [
             (execShellWith [ config.services.consul.package ] ''
-              consul kv put ${kvPath.node2} $(consul kv get ${kvPath.node1})
+              consul kv put intermediate/data $(consul kv get ${kvPath.node1})
               consul kv delete ${kvPath.node1}
             '')
+            (execShellWith [ config.services.consul.package ] ''
+              consul kv put ${kvPath.node2} $(consul kv get intermediate/data)
+              consul kv delete intermediate/data
+            '')
           ];
         };
       };

From d51e5e4847886069dd8b3778f4b8b995be55daee Mon Sep 17 00:00:00 2001
From: Max <max@privatevoid.net>
Date: Sun, 3 Sep 2023 21:19:50 +0200
Subject: [PATCH 5/6] modules/ascensions: allow specifying custom incantations

---
 modules/ascensions/default.nix | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/modules/ascensions/default.nix b/modules/ascensions/default.nix
index 4fe9dd8..4503512 100644
--- a/modules/ascensions/default.nix
+++ b/modules/ascensions/default.nix
@@ -34,7 +34,7 @@ let
     };
   };
 
-  runIncantations = f: with pkgs; f rec {
+  builtinIncantations = with pkgs; rec {
     execShellWith = extraPackages: script: writeShellScript "incantation" ''
       export PATH='${makeBinPath ([ coreutils ] ++ extraPackages)}'
       ${script}
@@ -53,13 +53,23 @@ let
     chmod = mode: target: "chmod -R ${escapeShellArgs [ mode target ]}";
   };
 
+  allIncantations = builtinIncantations // mapAttrs (_: mk: mk allIncantations) config.system.extraIncantations;
+
+  runIncantations = f: f allIncantations;
+
   consul = config.services.consul.package;
 in
 
 {
-  options.system.ascensions = mkOption {
-    type = with types; attrsOf (submodule ascensionType);
-    default = {};
+  options.system = {
+    ascensions = mkOption {
+      type = with types; attrsOf (submodule ascensionType);
+      default = {};
+    };
+    extraIncantations = mkOption {
+      type = with types; attrsOf (functionTo raw);
+      default = {};
+    };
   };
 
   config = {

From a6de5e2e3db44889e00a76126cf5073a9e45b4c7 Mon Sep 17 00:00:00 2001
From: Max <max@privatevoid.net>
Date: Sun, 3 Sep 2023 21:34:36 +0200
Subject: [PATCH 6/6] checks: only run on x86_64-linux

---
 packages/checks/default.nix | 6 +++---
 packages/system-filter.nix  | 8 --------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/packages/checks/default.nix b/packages/checks/default.nix
index ffb02d6..8b84917 100644
--- a/packages/checks/default.nix
+++ b/packages/checks/default.nix
@@ -1,8 +1,8 @@
-{ config, self, ... }:
+{ config, lib, self, ... }:
 
 {
-  perSystem = { filters, pkgs, self', ... }: {
-    checks = filters.doFilter filters.checks {
+  perSystem = { filters, pkgs, self', system, ... }: {
+    checks = lib.mkIf (system == "x86_64-linux") {
       ascensions = pkgs.callPackage ./ascensions.nix {
         inherit (self) nixosModules;
       };
diff --git a/packages/system-filter.nix b/packages/system-filter.nix
index f66cffb..151b853 100644
--- a/packages/system-filter.nix
+++ b/packages/system-filter.nix
@@ -11,12 +11,4 @@
     searxng = [ "x86_64-linux" ];
     tempo = [ "x86_64-linux" ];
   };
-  checks = {
-    ascensions = [ "x86_64-linux" ];
-    jellyfin-stateless = [ "x86_64-linux" ];
-    keycloak = [ "x86_64-linux" ];
-    patroni = [ "x86_64-linux" ];
-    searxng = [ "x86_64-linux" ];
-    tempo = [ "x86_64-linux" ];
-  };
 }