checks/simulacrum: rename from cluster, add more stuff, WIP

cluster/lib/service-module.nix

@@ -23,12 +23,12 @@ in
           * X evaluators, Y smallBuilders, Z bigBuilders
         etc.
       '';
-      type = with types; attrsOf (oneOf [ str (listOf str) ]);
+      type = with types; lazyAttrsOf (oneOf [ str (listOf str) ]);
       default = [];
     };
     otherNodes = mkOption {
       description = "Other nodes in the group.";
-      type = with types; attrsOf (functionTo (listOf str));
+      type = with types; lazyAttrsOf (functionTo (listOf str));
       default = [];
     };
     nixos = mkOption {

cluster/services/chant/listener.nix

@@ -1,9 +1,7 @@
 { config, lib, pkgs, ... }:
 
 let
-  consulCfg = config.services.consul.extraConfig;
-  consulIpAddr = consulCfg.addresses.http or "127.0.0.1";
-  consulHttpAddr = "${consulIpAddr}:${toString (consulCfg.ports.http or 8500)}";
+  consul = config.links.consulAgent;
 
   validTargets = lib.pipe config.systemd.services [
     (lib.filterAttrs (name: value: value.chant.enable))
@@ -62,8 +60,8 @@ in
   systemd.services.chant-listener = {
     description = "Chant Listener";
     wantedBy = [ "multi-user.target" ];
-    wants = [ "consul.service" ];
-    after = [ "consul.service" ];
+    requires = [ "consul-ready.service" ];
+    after = [ "consul-ready.service" ];
     serviceConfig = {
       ExecStart = "${config.services.consul.package}/bin/consul watch --type=event ${eventHandler}";
 
@@ -75,10 +73,10 @@ in
       RestartSec = 60;
       Restart = "always";
       IPAddressDeny = [ "any" ];
-      IPAddressAllow = [ consulIpAddr ];
+      IPAddressAllow = [ consul.ipv4 ];
     };
     environment = {
-      CONSUL_HTTP_ADDR = consulHttpAddr;
+      CONSUL_HTTP_ADDR = consul.tuple;
     };
   };
 }

cluster/services/consul/agent.nix

@@ -10,6 +10,8 @@ let
 in
 
 {
+  links.consulAgent.protocol = "http";
+
   services.consul = {
     enable = true;
     webUi = true;
@@ -24,11 +26,14 @@ in
       ports.serf_lan = hl.port;
       retry_join = map (hostName: hostLinks.${hostName}.consul.tuple) (cfg.otherNodes.agent hostName);
       bootstrap_expect = builtins.length cfg.nodes.agent;
+      addresses.http = config.links.consulAgent.ipv4;
+      ports.http = config.links.consulAgent.port;
     };
   };
 
   services.grafana-agent.settings.integrations.consul_exporter = {
     enabled = true;
     instance = hostName;
+    server = config.links.consulAgent.url;
   };
 }

cluster/services/consul/default.nix

@@ -15,6 +15,7 @@ in
     nixos.agent = [
       ./agent.nix
       ./remote-api.nix
+      ./ready.nix
     ];
   };
 

cluster/services/consul/ready.nix

@@ -0,0 +1,54 @@
+{ config, lib, pkgs, ... }:
+
+let
+  consulReady = pkgs.writers.writeHaskellBin "consul-ready" {
+    libraries = with pkgs.haskellPackages; [ aeson http-conduit watchdog ];
+  } ''
+    {-# LANGUAGE OverloadedStrings #-}
+    import Control.Watchdog
+    import Control.Exception
+    import System.IO
+    import Network.HTTP.Simple
+    import Data.Aeson
+
+    flushLogger :: WatchdogLogger String
+    flushLogger taskErr delay = do
+      defaultLogger taskErr delay
+      hFlush stdout
+
+    data ConsulHealth = ConsulHealth {
+      healthy :: Bool
+    }
+
+    instance FromJSON ConsulHealth where
+      parseJSON (Object v) = ConsulHealth <$> (v .: "Healthy")
+
+    handleException ex = case ex of
+      (SomeException _) -> return $ Left "Consul is not active"
+
+    main :: IO ()
+    main = watchdog $ do
+      setInitialDelay 300_000
+      setMaximumDelay 30_000_000
+      setLoggingAction flushLogger
+      watch $ handle handleException $ do
+          res <- httpJSON "${config.links.consulAgent.url}/v1/operator/autopilot/health"
+          case getResponseBody res of
+            ConsulHealth True -> return $ Right ()
+            ConsulHealth False -> return $ Left "Consul is unhealthy"
+  '';
+in
+
+{
+  systemd.services.consul-ready = {
+    description = "Wait for Consul";
+    requires = [ "consul.service" ];
+    after = [ "consul.service" ];
+    serviceConfig = {
+      ExecStart = lib.getExe consulReady;
+      DynamicUser = true;
+      TimeoutStartSec = "5m";
+      Type = "oneshot";
+    };
+  };
+}

cluster/services/consul/remote-api.nix

@@ -8,7 +8,7 @@ let
 in
 
 {
-  services.nginx.virtualHosts.${frontendDomain} = depot.lib.nginx.vhosts.proxy "http://127.0.0.1:8500" // {
+  services.nginx.virtualHosts.${frontendDomain} = depot.lib.nginx.vhosts.proxy config.links.consulAgent.url // {
     listenAddresses = lib.singleton addr;
     enableACME = false;
     useACMEHost = "internal.${domain}";
@@ -33,7 +33,7 @@ in
         {
           name = "Backend";
           id = "service:consul-remote:backend";
-          http = "http://127.0.0.1:8500/v1/status/leader";
+          http = "${config.links.consulAgent.url}/v1/status/leader";
           interval = "30s";
         }
       ];

cluster/services/forge/default.nix

@@ -30,4 +30,8 @@
     address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
     module = "https2xx";
   };
+
+  dns.records."ssh.forge".target = map
+    (node: depot.hours.${node}.interfaces.primary.addrPublic)
+    config.services.forge.nodes.server;
 }

cluster/services/forge/server.nix

@@ -51,6 +51,7 @@ in
         PROTOCOL = link.protocol;
         HTTP_ADDR = link.ipv4;
         HTTP_PORT = link.port;
+        SSH_DOMAIN = "ssh.${host}";
       };
       oauth2_client = {
         REGISTER_EMAIL_CONFIRM = false;
@@ -70,7 +71,6 @@ in
         MINIO_BUCKET = "forgejo";
         MINIO_USE_SSL = true;
         MINIO_BUCKET_LOOKUP = "path";
-        SERVE_DIRECT = true;
       };
       log."logger.xorm.MODE" = "";
       # enabling this will leak secrets to the log

cluster/services/locksmith/receiver.nix

@@ -1,9 +1,7 @@
 { config, lib, pkgs, ... }:
 
 let
-  consulCfg = config.services.consul.extraConfig;
-  consulIpAddr = consulCfg.addresses.http or "127.0.0.1";
-  consulHttpAddr = "${consulIpAddr}:${toString (consulCfg.ports.http or 8500)}";
+  consul = config.links.consulAgent;
 
   kvRoot = "secrets/locksmith";
   kvValue = "recipient/${config.networking.hostName}";
@@ -54,20 +52,20 @@ in
       systemd.services.locksmith = {
         description = "The Locksmith's Chant";
         wantedBy = [ "multi-user.target" ];
-        wants = [ "consul.service" ];
-        after = [ "consul.service" ];
+        requires = [ "consul-ready.service" ];
+        after = [ "consul-ready.service" ];
         chant.enable = true;
         path = [
           config.services.consul.package
         ];
         environment = {
-          CONSUL_HTTP_ADDR = consulHttpAddr;
+          CONSUL_HTTP_ADDR = consul.tuple;
         };
         serviceConfig = {
           PrivateTmp = true;
           WorkingDirectory = "/tmp";
           IPAddressDeny = [ "any" ];
-          IPAddressAllow = [ consulIpAddr ];
+          IPAddressAllow = [ consul.ipv4 ];
           LoadCredential = lib.mkForce [];
         };
         script = ''

cluster/services/patroni/worker.nix

@@ -41,7 +41,7 @@ in
     softwareWatchdog = true;
     settings = {
       consul = {
-        host = "127.0.0.1:8500";
+        host = config.links.consulAgent.tuple;
         register_service = true;
       };
       bootstrap.dcs = {

cluster/services/storage/default.nix

@@ -11,6 +11,7 @@ in
 
   services.storage = {
     nodes = {
+      internal = lib.subtractLists config.services.storage.nodes.external (lib.attrNames depot.gods.fromLight);
       external = [ "prophet" ];
       heresy = [ "VEGAS" ];
       garage = [ "grail" "prophet" "VEGAS" ];
@@ -19,6 +20,9 @@ in
       garageExternal = [ "grail" "prophet" ];
     };
     nixos = {
+      internal = [
+        ./internal.nix
+      ];
       external = [
         ./external.nix
         ./s3ql-upgrades.nix

cluster/services/storage/garage.nix

@@ -38,7 +38,7 @@ in
       rpc_public_addr = links.garageRpc.tuple;
       rpc_secret_file = config.age.secrets.garageRpcSecret.path;
       consul_discovery = {
-        consul_http_addr = "http://127.0.0.1:8500";
+        consul_http_addr = config.links.consulAgent.url;
         service_name = "garage-discovery";
       };
       s3_api = {

cluster/services/storage/internal.nix

@@ -0,0 +1,12 @@
+{ ... }:
+
+let
+  storageDir = "/srv/storage";
+in
+
+{
+  systemd.tmpfiles.settings."00-storage" = {
+    "${storageDir}".d.mode = "0755";
+    "${storageDir}/private".d.mode = "0751";
+  };
+}

cluster/services/ways/host.nix

@@ -4,8 +4,6 @@ let
   externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;
 
   consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
-
-  consulHttpAddr = "${config.services.consul.extraConfig.addresses.http or "127.0.0.1"}:${toString (config.services.consul.extraConfig.ports.http or 8500)}";
 in
 
 {
@@ -63,7 +61,7 @@ in
     user = "nginx";
     group = "nginx";
     settings = {
-      consul.address = "http://${consulHttpAddr}";
+      consul.address = config.links.consulAgent.url;
       template = [
         {
           source = let

modules/ascensions/default.nix

@@ -87,7 +87,8 @@ in
           description = "Ascension for ${name}";
           wantedBy = [ "multi-user.target" ];
           inherit (asc) requiredBy before;
-          after = asc.after ++ (lib.optional asc.distributed "consul.service");
+          after = asc.after ++ (lib.optional asc.distributed "consul-ready.service");
+          requires = lib.optional asc.distributed "consul-ready.service";
           serviceConfig.Type = "oneshot";
           distributed.enable = asc.distributed;
           script = ''

modules/consul-distributed-services/default.nix

@@ -42,6 +42,10 @@ in
 
       hasSpecialPrefix = elem (substring 0 1 ExecStart) [ "@" "-" ":" "+" "!" ];
     in assert !hasSpecialPrefix; pkgs.writeTextDir "etc/systemd/system/${n}.service.d/distributed.conf" ''
+      [Unit]
+      Requires=consul-ready.service
+      After=consul-ready.service
+
       [Service]
       ExecStartPre=${waitForConsul} 'services/${n}%i'
       ExecStart=

modules/consul-service-registry/default.nix

@@ -81,14 +81,16 @@ let
     }.${mode};
     value = {
       direct = {
-        after = [ "consul.service" ];
+        after = [ "consul-ready.service" ];
+        requires = [ "consul-ready.service" ];
         serviceConfig = {
           ExecStartPost = register servicesJson;
           ExecStopPost = deregister servicesJson;
         };
       };
       external = {
-        after = [ "consul.service" "${unit}.service" ];
+        after = [ "consul-ready.service" "${unit}.service" ];
+        requires = [ "consul-ready.service" ];
         wantedBy = [ "${unit}.service" ];
         unitConfig.BindsTo = "${unit}.service";
         serviceConfig = {

packages/checks/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, self, ... }:
+{ config, lib, self, extendModules, ... }:
 
 let
   timeMachine = {
@@ -7,6 +7,7 @@ let
 in
 
 {
+  debug = lib.warn "debug mode is enabled" true;
   perSystem = { filters, pkgs, self', system, ... }: {
     checks = lib.mkIf (system == "x86_64-linux") {
       ascensions = pkgs.callPackage ./ascensions.nix {
@@ -48,6 +49,10 @@ in
       searxng = pkgs.callPackage ./searxng.nix {
         inherit (self'.packages) searxng;
       };
+
+      simulacrum = pkgs.callPackage ./simulacrum.nix {
+        inherit config extendModules;
+      };
     };
   };
 }

packages/checks/modules/nixos/external-storage.nix

@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+
+{
+  systemd.tmpfiles.settings."00-testing-external-storage-underlays" = lib.mapAttrs' (name: cfg: {
+    name = cfg.mountpoint;
+    value.d = {
+      user = toString cfg.uid;
+      group = toString cfg.gid;
+      mode = "0700";
+    };
+  }) config.services.external-storage.underlays;
+}

packages/checks/simulacrum.nix

@@ -0,0 +1,87 @@
+{ testers, config, extendModules, lib, system }:
+
+let
+  lift = config;
+
+  snakeoil = {
+    ssh = {
+      public = lib.fileContents ./snakeoil/ssh/snakeoil-key.pub;
+      private = ./snakeoil/ssh/snakeoil-key;
+    };
+    wireguard = {
+      public = lib.genAttrs nodes (node: lib.fileContents ./snakeoil/wireguard/public-key-${toString digits.${node}});
+      private = lib.genAttrs nodes (node: ./snakeoil/wireguard/private-key-${toString digits.${node}});
+    };
+  };
+
+  nodes = lib.attrNames config.gods.fromLight;
+  digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes (lib.range 1 255));
+  depot' = extendModules {
+    modules = [
+      ({ config, ... }: {
+        gods.fromLight = lib.mapAttrs (name: cfg: {
+          interfaces.primary = {
+            link = lib.mkForce "eth1";
+            addr = lib.mkForce "192.168.1.${toString digits.${name}}";
+            addrPublic = lib.mkForce "192.168.1.${toString digits.${name}}";
+          };
+          ssh.id.publicKey = lib.mkForce snakeoil.ssh.public;
+        }) lift.gods.fromLight;
+
+        cluster = lib.mkForce (lift.cluster.extendModules {
+          specialArgs.depot = config;
+          modules = [
+            {
+              hostLinks = lib.genAttrs nodes (node: {
+                mesh.extra = lib.mkForce (lift.cluster.config.hostLinks.${node}.mesh.extra // {
+                  pubKey = snakeoil.wireguard.public.${node};
+                });
+              });
+            }
+          ];
+        });
+      })
+    ];
+  };
+  specialArgs = depot'.config.lib.summon system lib.id;
+in
+
+testers.runNixOSTest {
+  name = "simulacrum";
+
+  node = { inherit specialArgs; };
+  nodes = lib.genAttrs nodes (node: {
+    imports = [
+      specialArgs.depot.hours.${node}.nixos
+      ./modules/nixos/age-dummy-secrets
+      ./modules/nixos/external-storage.nix
+    ] ++ depot'.config.cluster.config.out.injectNixosConfig node;
+
+    systemd.services = {
+      hyprspace.enable = false;
+      cachix-agent.enable = false;
+    };
+
+    environment.etc = {
+      "ssh/ssh_host_ed25519_key" = {
+        source = snakeoil.ssh.private;
+        mode = "0400";
+      };
+      "dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce snakeoil.wireguard.private.${node};
+      "dummy-secrets/grafana-agent-blackbox-secret-monitoring".text = lib.mkForce ''
+        SECRET_MONITORING_BLACKBOX_TARGET_1_NAME=example-external-service
+        SECRET_MONITORING_BLACKBOX_TARGET_1_MODULE=http2xx
+        SECRET_MONITORING_BLACKBOX_TARGET_1_ADDRESS=http://127.0.0.1:1
+      '';
+      "dummy-secrets/garageRpcSecret".text = lib.mkForce "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+    };
+    virtualisation = {
+      cores = 2;
+      memorySize = 4096;
+    };
+  });
+
+  testScript = ''
+    grail.succeed("false")
+  '';
+}

packages/checks/snakeoil/ssh/snakeoil-key

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgwAAAJAS78fWEu/H
+1gAAAAtzc2gtZWQyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgw
+AAAEAUtGOZZIZdzGP6g85JuXBjDtciNQ9bLHNxSN5Gbwvb2Q7HTdf4u1bRo3x6N03ggmAM
++tNmBXB7tZteGEG+pXCDAAAACW1heEBUSVRBTgECAwQ=
+-----END OPENSSH PRIVATE KEY-----

packages/checks/snakeoil/ssh/snakeoil-key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA7HTdf4u1bRo3x6N03ggmAM+tNmBXB7tZteGEG+pXCD

packages/checks/snakeoil/wireguard/private-key-1

@@ -0,0 +1 @@
+YHzP8rBP6qiXs6ZdnvHop9KnCYRADIEejwZzAzvj8m4=

packages/checks/snakeoil/wireguard/private-key-2

@@ -0,0 +1 @@
+uD7X5E6N9d0sN+xPr/bWnehSa3bAok741GO7Z4I+Z3I=

packages/checks/snakeoil/wireguard/private-key-3

@@ -0,0 +1 @@
+YLl+hkWaCWx/5PpWs3cQ+bKqYdJef/qZ+FMTsM9ammM=

packages/checks/snakeoil/wireguard/private-key-4

@@ -0,0 +1 @@
+MNvWpMluuzQvPyGTp7jtyPSyz6n9lIly/WX1gW2NAHg=

packages/checks/snakeoil/wireguard/private-key-5

@@ -0,0 +1 @@
+QHyIJ3HoKGGFN28qOrQP4UyoQMP5bM7Idn2MzayKzEM=

packages/checks/snakeoil/wireguard/public-key-1

@@ -0,0 +1 @@
+TESTtbFybW5YREwtd18a1A4StS4YAIUS5/M1Lv0jHjA=

packages/checks/snakeoil/wireguard/public-key-2

@@ -0,0 +1 @@
+TEsTh7bthkaDh9A1CpqDi/F121ao5lRZqIJznLH8mB4=

packages/checks/snakeoil/wireguard/public-key-3

@@ -0,0 +1 @@
+tEST6afFmVN18o+EiWNFx+ax3MJwdQIeNfJSGEpffXw=

packages/checks/snakeoil/wireguard/public-key-4

@@ -0,0 +1 @@
+tEsT6s7VtM5C20eJBaq6UlQydAha8ATlmrTRe9T5jnM=

packages/checks/snakeoil/wireguard/public-key-5

@@ -0,0 +1 @@
+TEstYyb5IoqSL53HbSQwMhTaR16sxcWcMmXIBPd+1gE=