cluster: use consul-ready when waiting for consul

cluster/services/chant/listener.nix

@@ -60,8 +60,8 @@ in
   systemd.services.chant-listener = {
     description = "Chant Listener";
     wantedBy = [ "multi-user.target" ];
-    wants = [ "consul.service" ];
-    after = [ "consul.service" ];
+    requires = [ "consul-ready.service" ];
+    after = [ "consul-ready.service" ];
     serviceConfig = {
       ExecStart = "${config.services.consul.package}/bin/consul watch --type=event ${eventHandler}";
 

cluster/services/consul/default.nix

@@ -15,6 +15,7 @@ in
     nixos.agent = [
       ./agent.nix
       ./remote-api.nix
+      ./ready.nix
     ];
   };
 

cluster/services/consul/ready.nix

@@ -0,0 +1,54 @@
+{ config, lib, pkgs, ... }:
+
+let
+  consulReady = pkgs.writers.writeHaskellBin "consul-ready" {
+    libraries = with pkgs.haskellPackages; [ aeson http-conduit watchdog ];
+  } ''
+    {-# LANGUAGE OverloadedStrings #-}
+    import Control.Watchdog
+    import Control.Exception
+    import System.IO
+    import Network.HTTP.Simple
+    import Data.Aeson
+
+    flushLogger :: WatchdogLogger String
+    flushLogger taskErr delay = do
+      defaultLogger taskErr delay
+      hFlush stdout
+
+    data ConsulHealth = ConsulHealth {
+      healthy :: Bool
+    }
+
+    instance FromJSON ConsulHealth where
+      parseJSON (Object v) = ConsulHealth <$> (v .: "Healthy")
+
+    handleException ex = case ex of
+      (SomeException _) -> return $ Left "Consul is not active"
+
+    main :: IO ()
+    main = watchdog $ do
+      setInitialDelay 300_000
+      setMaximumDelay 30_000_000
+      setLoggingAction flushLogger
+      watch $ handle handleException $ do
+          res <- httpJSON "${config.links.consulAgent.url}/v1/operator/autopilot/health"
+          case getResponseBody res of
+            ConsulHealth True -> return $ Right ()
+            ConsulHealth False -> return $ Left "Consul is unhealthy"
+  '';
+in
+
+{
+  systemd.services.consul-ready = {
+    description = "Wait for Consul";
+    requires = [ "consul.service" ];
+    after = [ "consul.service" ];
+    serviceConfig = {
+      ExecStart = lib.getExe consulReady;
+      DynamicUser = true;
+      TimeoutStartSec = "5m";
+      Type = "oneshot";
+    };
+  };
+}

cluster/services/locksmith/receiver.nix

@@ -52,8 +52,8 @@ in
       systemd.services.locksmith = {
         description = "The Locksmith's Chant";
         wantedBy = [ "multi-user.target" ];
-        wants = [ "consul.service" ];
-        after = [ "consul.service" ];
+        requires = [ "consul-ready.service" ];
+        after = [ "consul-ready.service" ];
         chant.enable = true;
         path = [
           config.services.consul.package

modules/ascensions/default.nix

@@ -87,7 +87,8 @@ in
           description = "Ascension for ${name}";
           wantedBy = [ "multi-user.target" ];
           inherit (asc) requiredBy before;
-          after = asc.after ++ (lib.optional asc.distributed "consul.service");
+          after = asc.after ++ (lib.optional asc.distributed "consul-ready.service");
+          requires = lib.optional asc.distributed "consul-ready.service";
           serviceConfig.Type = "oneshot";
           distributed.enable = asc.distributed;
           script = ''

modules/consul-distributed-services/default.nix

@@ -42,6 +42,10 @@ in
 
       hasSpecialPrefix = elem (substring 0 1 ExecStart) [ "@" "-" ":" "+" "!" ];
     in assert !hasSpecialPrefix; pkgs.writeTextDir "etc/systemd/system/${n}.service.d/distributed.conf" ''
+      [Unit]
+      Requires=consul-ready.service
+      After=consul-ready.service
+
       [Service]
       ExecStartPre=${waitForConsul} 'services/${n}%i'
       ExecStart=

modules/consul-service-registry/default.nix

@@ -81,14 +81,16 @@ let
     }.${mode};
     value = {
       direct = {
-        after = [ "consul.service" ];
+        after = [ "consul-ready.service" ];
+        requires = [ "consul-ready.service" ];
         serviceConfig = {
           ExecStartPost = register servicesJson;
           ExecStopPost = deregister servicesJson;
         };
       };
       external = {
-        after = [ "consul.service" "${unit}.service" ];
+        after = [ "consul-ready.service" "${unit}.service" ];
+        requires = [ "consul-ready.service" ];
         wantedBy = [ "${unit}.service" ];
         unitConfig.BindsTo = "${unit}.service";
         serviceConfig = {