checks/cluster: init WIP

cluster/lib/service-module.nix

@@ -23,12 +23,12 @@ in
           * X evaluators, Y smallBuilders, Z bigBuilders
         etc.
       '';
-      type = with types; lazyAttrsOf (oneOf [ str (listOf str) ]);
+      type = with types; attrsOf (oneOf [ str (listOf str) ]);
       default = [];
     };
     otherNodes = mkOption {
       description = "Other nodes in the group.";
-      type = with types; lazyAttrsOf (functionTo (listOf str));
+      type = with types; attrsOf (functionTo (listOf str));
       default = [];
     };
     nixos = mkOption {

cluster/services/chant/listener.nix

@@ -1,7 +1,9 @@
 { config, lib, pkgs, ... }:
 
 let
-  consul = config.links.consulAgent;
+  consulCfg = config.services.consul.extraConfig;
+  consulIpAddr = consulCfg.addresses.http or "127.0.0.1";
+  consulHttpAddr = "${consulIpAddr}:${toString (consulCfg.ports.http or 8500)}";
 
   validTargets = lib.pipe config.systemd.services [
     (lib.filterAttrs (name: value: value.chant.enable))
@@ -60,8 +62,8 @@ in
   systemd.services.chant-listener = {
     description = "Chant Listener";
     wantedBy = [ "multi-user.target" ];
-    requires = [ "consul-ready.service" ];
-    after = [ "consul-ready.service" ];
+    wants = [ "consul.service" ];
+    after = [ "consul.service" ];
     serviceConfig = {
       ExecStart = "${config.services.consul.package}/bin/consul watch --type=event ${eventHandler}";
 
@@ -73,10 +75,10 @@ in
       RestartSec = 60;
       Restart = "always";
       IPAddressDeny = [ "any" ];
-      IPAddressAllow = [ consul.ipv4 ];
+      IPAddressAllow = [ consulIpAddr ];
     };
     environment = {
-      CONSUL_HTTP_ADDR = consul.tuple;
+      CONSUL_HTTP_ADDR = consulHttpAddr;
     };
   };
 }

cluster/services/consul/agent.nix

@@ -10,8 +10,6 @@ let
 in
 
 {
-  links.consulAgent.protocol = "http";
-
   services.consul = {
     enable = true;
     webUi = true;
@@ -26,14 +24,11 @@ in
       ports.serf_lan = hl.port;
       retry_join = map (hostName: hostLinks.${hostName}.consul.tuple) (cfg.otherNodes.agent hostName);
       bootstrap_expect = builtins.length cfg.nodes.agent;
-      addresses.http = config.links.consulAgent.ipv4;
-      ports.http = config.links.consulAgent.port;
     };
   };
 
   services.grafana-agent.settings.integrations.consul_exporter = {
     enabled = true;
     instance = hostName;
-    server = config.links.consulAgent.url;
   };
 }

cluster/services/consul/default.nix

@@ -15,7 +15,6 @@ in
     nixos.agent = [
       ./agent.nix
       ./remote-api.nix
-      ./ready.nix
     ];
   };
 

cluster/services/consul/ready.nix

@@ -1,54 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  consulReady = pkgs.writers.writeHaskellBin "consul-ready" {
-    libraries = with pkgs.haskellPackages; [ aeson http-conduit watchdog ];
-  } ''
-    {-# LANGUAGE OverloadedStrings #-}
-    import Control.Watchdog
-    import Control.Exception
-    import System.IO
-    import Network.HTTP.Simple
-    import Data.Aeson
-
-    flushLogger :: WatchdogLogger String
-    flushLogger taskErr delay = do
-      defaultLogger taskErr delay
-      hFlush stdout
-
-    data ConsulHealth = ConsulHealth {
-      healthy :: Bool
-    }
-
-    instance FromJSON ConsulHealth where
-      parseJSON (Object v) = ConsulHealth <$> (v .: "Healthy")
-
-    handleException ex = case ex of
-      (SomeException _) -> return $ Left "Consul is not active"
-
-    main :: IO ()
-    main = watchdog $ do
-      setInitialDelay 300_000
-      setMaximumDelay 30_000_000
-      setLoggingAction flushLogger
-      watch $ handle handleException $ do
-          res <- httpJSON "${config.links.consulAgent.url}/v1/operator/autopilot/health"
-          case getResponseBody res of
-            ConsulHealth True -> return $ Right ()
-            ConsulHealth False -> return $ Left "Consul is unhealthy"
-  '';
-in
-
-{
-  systemd.services.consul-ready = {
-    description = "Wait for Consul";
-    requires = [ "consul.service" ];
-    after = [ "consul.service" ];
-    serviceConfig = {
-      ExecStart = lib.getExe consulReady;
-      DynamicUser = true;
-      TimeoutStartSec = "5m";
-      Type = "oneshot";
-    };
-  };
-}

cluster/services/consul/remote-api.nix

@@ -8,7 +8,7 @@ let
 in
 
 {
-  services.nginx.virtualHosts.${frontendDomain} = depot.lib.nginx.vhosts.proxy config.links.consulAgent.url // {
+  services.nginx.virtualHosts.${frontendDomain} = depot.lib.nginx.vhosts.proxy "http://127.0.0.1:8500" // {
     listenAddresses = lib.singleton addr;
     enableACME = false;
     useACMEHost = "internal.${domain}";
@@ -33,7 +33,7 @@ in
         {
           name = "Backend";
           id = "service:consul-remote:backend";
-          http = "${config.links.consulAgent.url}/v1/status/leader";
+          http = "http://127.0.0.1:8500/v1/status/leader";
           interval = "30s";
         }
       ];

cluster/services/forge/default.nix

@@ -30,8 +30,4 @@
     address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
     module = "https2xx";
   };
-
-  dns.records."ssh.forge".target = map
-    (node: depot.hours.${node}.interfaces.primary.addrPublic)
-    config.services.forge.nodes.server;
 }

cluster/services/forge/server.nix

@@ -51,7 +51,6 @@ in
         PROTOCOL = link.protocol;
         HTTP_ADDR = link.ipv4;
         HTTP_PORT = link.port;
-        SSH_DOMAIN = "ssh.${host}";
       };
       oauth2_client = {
         REGISTER_EMAIL_CONFIRM = false;
@@ -71,6 +70,7 @@ in
         MINIO_BUCKET = "forgejo";
         MINIO_USE_SSL = true;
         MINIO_BUCKET_LOOKUP = "path";
+        SERVE_DIRECT = true;
       };
       log."logger.xorm.MODE" = "";
       # enabling this will leak secrets to the log

cluster/services/locksmith/receiver.nix

@@ -1,7 +1,9 @@
 { config, lib, pkgs, ... }:
 
 let
-  consul = config.links.consulAgent;
+  consulCfg = config.services.consul.extraConfig;
+  consulIpAddr = consulCfg.addresses.http or "127.0.0.1";
+  consulHttpAddr = "${consulIpAddr}:${toString (consulCfg.ports.http or 8500)}";
 
   kvRoot = "secrets/locksmith";
   kvValue = "recipient/${config.networking.hostName}";
@@ -52,20 +54,20 @@ in
       systemd.services.locksmith = {
         description = "The Locksmith's Chant";
         wantedBy = [ "multi-user.target" ];
-        requires = [ "consul-ready.service" ];
-        after = [ "consul-ready.service" ];
+        wants = [ "consul.service" ];
+        after = [ "consul.service" ];
         chant.enable = true;
         path = [
           config.services.consul.package
         ];
         environment = {
-          CONSUL_HTTP_ADDR = consul.tuple;
+          CONSUL_HTTP_ADDR = consulHttpAddr;
         };
         serviceConfig = {
           PrivateTmp = true;
           WorkingDirectory = "/tmp";
           IPAddressDeny = [ "any" ];
-          IPAddressAllow = [ consul.ipv4 ];
+          IPAddressAllow = [ consulIpAddr ];
           LoadCredential = lib.mkForce [];
         };
         script = ''

cluster/services/patroni/worker.nix

@@ -41,7 +41,7 @@ in
     softwareWatchdog = true;
     settings = {
       consul = {
-        host = config.links.consulAgent.tuple;
+        host = "127.0.0.1:8500";
         register_service = true;
       };
       bootstrap.dcs = {

cluster/services/storage/default.nix

@@ -11,7 +11,6 @@ in
 
   services.storage = {
     nodes = {
-      internal = lib.subtractLists config.services.storage.nodes.external (lib.attrNames depot.gods.fromLight);
       external = [ "prophet" ];
       heresy = [ "VEGAS" ];
       garage = [ "grail" "prophet" "VEGAS" ];
@@ -20,9 +19,6 @@ in
       garageExternal = [ "grail" "prophet" ];
     };
     nixos = {
-      internal = [
-        ./internal.nix
-      ];
       external = [
         ./external.nix
         ./s3ql-upgrades.nix

cluster/services/storage/garage.nix

@@ -38,7 +38,7 @@ in
       rpc_public_addr = links.garageRpc.tuple;
       rpc_secret_file = config.age.secrets.garageRpcSecret.path;
       consul_discovery = {
-        consul_http_addr = config.links.consulAgent.url;
+        consul_http_addr = "http://127.0.0.1:8500";
         service_name = "garage-discovery";
       };
       s3_api = {

cluster/services/storage/internal.nix

@@ -1,12 +0,0 @@
-{ ... }:
-
-let
-  storageDir = "/srv/storage";
-in
-
-{
-  systemd.tmpfiles.settings."00-storage" = {
-    "${storageDir}".d.mode = "0755";
-    "${storageDir}/private".d.mode = "0751";
-  };
-}

cluster/services/ways/host.nix

@@ -4,6 +4,8 @@ let
   externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;
 
   consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
+
+  consulHttpAddr = "${config.services.consul.extraConfig.addresses.http or "127.0.0.1"}:${toString (config.services.consul.extraConfig.ports.http or 8500)}";
 in
 
 {
@@ -61,7 +63,7 @@ in
     user = "nginx";
     group = "nginx";
     settings = {
-      consul.address = config.links.consulAgent.url;
+      consul.address = "http://${consulHttpAddr}";
       template = [
         {
           source = let

modules/ascensions/default.nix

@@ -87,8 +87,7 @@ in
           description = "Ascension for ${name}";
           wantedBy = [ "multi-user.target" ];
           inherit (asc) requiredBy before;
-          after = asc.after ++ (lib.optional asc.distributed "consul-ready.service");
-          requires = lib.optional asc.distributed "consul-ready.service";
+          after = asc.after ++ (lib.optional asc.distributed "consul.service");
           serviceConfig.Type = "oneshot";
           distributed.enable = asc.distributed;
           script = ''

modules/consul-distributed-services/default.nix

@@ -42,10 +42,6 @@ in
 
       hasSpecialPrefix = elem (substring 0 1 ExecStart) [ "@" "-" ":" "+" "!" ];
     in assert !hasSpecialPrefix; pkgs.writeTextDir "etc/systemd/system/${n}.service.d/distributed.conf" ''
-      [Unit]
-      Requires=consul-ready.service
-      After=consul-ready.service
-
       [Service]
       ExecStartPre=${waitForConsul} 'services/${n}%i'
       ExecStart=

modules/consul-service-registry/default.nix

@@ -81,16 +81,14 @@ let
     }.${mode};
     value = {
       direct = {
-        after = [ "consul-ready.service" ];
-        requires = [ "consul-ready.service" ];
+        after = [ "consul.service" ];
         serviceConfig = {
           ExecStartPost = register servicesJson;
           ExecStopPost = deregister servicesJson;
         };
       };
       external = {
-        after = [ "consul-ready.service" "${unit}.service" ];
-        requires = [ "consul-ready.service" ];
+        after = [ "consul.service" "${unit}.service" ];
         wantedBy = [ "${unit}.service" ];
         unitConfig.BindsTo = "${unit}.service";
         serviceConfig = {

packages/checks/cluster.nix

@@ -47,38 +47,22 @@ let
 in
 
 testers.runNixOSTest {
-  name = "simulacrum";
+  name = "cluster";
 
   node = { inherit specialArgs; };
   nodes = lib.genAttrs nodes (node: {
     imports = [
       specialArgs.depot.hours.${node}.nixos
       ./modules/nixos/age-dummy-secrets
-      ./modules/nixos/external-storage.nix
     ] ++ depot'.config.cluster.config.out.injectNixosConfig node;
 
-    systemd.services = {
-      hyprspace.enable = false;
-      cachix-agent.enable = false;
-    };
-
-    environment.etc = {
-      "ssh/ssh_host_ed25519_key" = {
-        source = snakeoil.ssh.private;
-        mode = "0400";
-      };
-      "dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce snakeoil.wireguard.private.${node};
-      "dummy-secrets/grafana-agent-blackbox-secret-monitoring".text = lib.mkForce ''
-        SECRET_MONITORING_BLACKBOX_TARGET_1_NAME=example-external-service
-        SECRET_MONITORING_BLACKBOX_TARGET_1_MODULE=http2xx
-        SECRET_MONITORING_BLACKBOX_TARGET_1_ADDRESS=http://127.0.0.1:1
-      '';
-      "dummy-secrets/garageRpcSecret".text = lib.mkForce "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
-    };
-    virtualisation = {
-      cores = 2;
-      memorySize = 4096;
+    environment.etc."ssh/ssh_host_ed25519_key" = {
+      source = snakeoil.ssh.private;
+      mode = "0400";
     };
+    environment.etc."dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce snakeoil.wireguard.private.${node};
+    passthru.depot = depot';
+    virtualisation.memorySize = 4096;
   });
 
   testScript = ''

packages/checks/default.nix

@@ -15,6 +15,10 @@ in
         inherit (self) nixosModules;
       };
 
+      cluster = pkgs.callPackage ./cluster.nix {
+        inherit config extendModules;
+      };
+
       garage = pkgs.callPackage ./garage.nix {
         inherit (self'.packages) garage consul;
         inherit (self) nixosModules;
@@ -49,10 +53,6 @@ in
       searxng = pkgs.callPackage ./searxng.nix {
         inherit (self'.packages) searxng;
       };
-
-      simulacrum = pkgs.callPackage ./simulacrum.nix {
-        inherit config extendModules;
-      };
     };
   };
 }

packages/checks/modules/nixos/external-storage.nix

@@ -1,12 +0,0 @@
-{ config, lib, ... }:
-
-{
-  systemd.tmpfiles.settings."00-testing-external-storage-underlays" = lib.mapAttrs' (name: cfg: {
-    name = cfg.mountpoint;
-    value.d = {
-      user = toString cfg.uid;
-      group = toString cfg.gid;
-      mode = "0700";
-    };
-  }) config.services.external-storage.underlays;
-}