cluster/default.nix

@@ -14,6 +14,7 @@ lib.evalModules {
     ./lib/services.nix
     ./lib/inject-nixos-config.nix
     ./lib/port-magic-multi.nix
+    ./lib/mesh.nix
 
     ./import-services.nix
   ];

cluster/lib/mesh.nix

@@ -0,0 +1,15 @@
+{ config, lib, ... }:
+
+{
+  hostLinks = lib.pipe config.services [
+    (lib.filterAttrs (_: svc: svc.meshLinks != {}))
+    (lib.mapAttrsToList (svcName: svc: lib.mapAttrsToList (name: cfg: lib.genAttrs svc.nodes.${name} (hostName: {
+      ${cfg.name} = { ... }: {
+        imports = [ cfg.link ];
+        ipv4 = config.vars.mesh.${hostName}.meshIp;
+      };
+    })) svc.meshLinks))
+    (map lib.mkMerge)
+    lib.mkMerge
+  ];
+}

cluster/lib/service-module.nix

@@ -1,8 +1,9 @@
-{ config, lib, ... }:
+{ config, lib, name, ... }:
 with lib;
 
 let
   filterGroup = group: hostName: builtins.filter (x: x != hostName) group;
+  serviceName = name;
 in
 
 {
@@ -31,6 +32,22 @@ in
       type = with types; attrs;
       default = {};
     };
+    meshLinks = mkOption {
+      description = "Create host links on the mesh network.";
+      type = types.attrsOf (types.submodule ({ name, ... }: {
+        options = {
+          name = mkOption {
+            type = types.str;
+            default = "${serviceName}-${name}";
+          };
+          link = mkOption {
+            type = types.deferredModule;
+            default = {};
+          };
+        };
+      }));
+      default = {};
+    };
   };
   config.otherNodes = builtins.mapAttrs (const filterGroup) config.nodes;
 }

cluster/services/forge/default.nix

@@ -4,11 +4,15 @@
   services.forge = {
     nodes.server = [ "VEGAS" ];
     nixos.server = ./server.nix;
+    meshLinks.server = {
+      name = "forge";
+      link.protocol = "http";
+    };
   };
 
-  dns.records.forge.target = map
+  ways.forge.target = let
-    (node: depot.hours.${node}.interfaces.primary.addrPublic)
+    host = builtins.head config.services.forge.nodes.server;
-    config.services.forge.nodes.server;
+  in config.hostLinks.${host}.forge.url;
 
   garage = {
     keys.forgejo = { };

cluster/services/forge/server.nix

@@ -9,7 +9,7 @@ let
 
   host = "forge.${domain}";
 
-  link = config.links.forge;
+  link = cluster.config.hostLinks.${config.networking.hostName}.forge;
 
   exe = lib.getExe config.services.forgejo.package;
 in
@@ -43,8 +43,6 @@ in
     };
   };
 
-  links.forge.protocol = "http";
-
   services.forgejo = {
     enable = true;
     package = depot.packages.forgejo;
@@ -101,8 +99,6 @@ in
     };
   };
 
-  services.nginx.virtualHosts."${host}" = vhosts.proxy link.url;
-
   systemd.services.forgejo.preStart = let
     providerName = "PrivateVoidAccount";
     args = lib.escapeShellArgs [

cluster/services/ipfs/default.nix

@@ -29,6 +29,10 @@
       io-tweaks = [ "VEGAS" ];
       remote-api = [ "VEGAS" ];
     };
+    meshLinks.gateway = {
+      name = "ipfsGateway";
+      link.protocol = "http";
+    };
     nixos = {
       node = [
         ./node.nix
@@ -51,18 +55,37 @@
   };
 
   dns.records = {
-    p2p.consulService = "ipfs-gateway";
-    pin.consulService = "ipfs-gateway";
     "ipfs.admin".target = map
       (node: depot.hours.${node}.interfaces.primary.addrPublic)
       config.services.ipfs.nodes.remote-api;
-    "^[^_].+\\.ipfs" = {
+    pin.consulService = "ipfs-gateway";
+  };
+
+  ways = {
+    p2p = {
       consulService = "ipfs-gateway";
-      rewrite.type = "regex";
+      extras.locations."/routing" = {
+        extraConfig = ''
+          add_header X-Content-Type-Options "";
+          add_header Access-Control-Allow-Origin *;
+        '';
+      };
     };
-    "^[^_].+\\.ipns" = {
+    ipfs = {
       consulService = "ipfs-gateway";
-      rewrite.type = "regex";
+      wildcard = true;
+      extras.extraConfig = ''
+        add_header X-Content-Type-Options "";
+        add_header Access-Control-Allow-Origin *;
+      '';
+    };
+    ipns = {
+      consulService = "ipfs-gateway";
+      wildcard = true;
+      extras.extraConfig = ''
+        add_header X-Content-Type-Options "";
+        add_header Access-Control-Allow-Origin *;
+      '';
     };
   };
 }

cluster/services/ipfs/gateway.nix

@@ -1,8 +1,7 @@
-{ config, depot, lib, ... }:
+{ cluster, config, depot, lib, ... }:
-with depot.lib.nginx;
 let
   inherit (depot.lib.meta) domain;
-  gw = config.links.ipfsGateway;
+  gw = cluster.config.hostLinks.${config.networking.hostName}.ipfsGateway;
   cfg = config.services.ipfs;
   metrics = config.links.ipfsMetrics;
 in
@@ -34,42 +33,6 @@ in
       locations."/".return = "204";
       locations."${metrics.path}".proxyPass = "http://unix:/run/ipfs/ipfs-api.sock:";
     };
-    "p2p.${domain}" = vhosts.basic // {
-      locations."/".return = "204";
-      locations."/routing" = {
-        proxyPass = gw.url;
-        extraConfig = ''
-          add_header X-Content-Type-Options "";
-          add_header Access-Control-Allow-Origin *;
-        '';
-      };
-    };
-  };
-  security.acme.certs."ipfs.${domain}" = {
-    domain = "*.ipfs.${domain}";
-    extraDomainNames = [ "*.ipns.${domain}" ];
-    dnsProvider = "exec";
-    group = "nginx";
-  };
-
-  security.acme.certs."p2p.${domain}" = {
-    dnsProvider = "exec";
-    webroot = lib.mkForce null;
-  };
-
-  services.nginx.virtualHosts."ipfs.${domain}" = vhosts.basic // {
-    serverName = "~^(.+)\.(ip[fn]s)\.${domain}$";
-    enableACME = false;
-    useACMEHost = "ipfs.${domain}";
-    locations = {
-      "/" = {
-        proxyPass = gw.url;
-        extraConfig = ''
-          add_header X-Content-Type-Options "";
-          add_header Access-Control-Allow-Origin *;
-        '';
-      };
-    };
   };
 
   services.ipfs.extraConfig.Gateway.PublicGateways = {
@@ -88,20 +51,11 @@ in
   consul.services.ipfs-gateway = {
     mode = "external";
     unit = "ipfs";
-    definition = rec {
+    definition = {
       name = "ipfs-gateway";
-      address = depot.reflection.interfaces.primary.addrPublic;
+      address = gw.ipv4;
-      port = 443;
+      port = gw.port;
       checks = [
-        rec {
-          name = "Frontend";
-          id = "service:ipfs-gateway:frontend";
-          interval = "60s";
-          http = "https://${address}/";
-          tls_server_name = "bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354.ipfs.${domain}"; # empty directory
-          header.Host = lib.singleton tls_server_name;
-          method = "HEAD";
-        }
         {
           name = "IPFS Node";
           id = "service:ipfs-gateway:ipfs";

cluster/services/ipfs/node.nix

@@ -4,7 +4,7 @@ let
   cfg = config.services.ipfs;
   apiAddress = "/unix/run/ipfs/ipfs-api.sock";
   ipfsApi = pkgs.writeTextDir "api" apiAddress;
-  gw = config.links.ipfsGateway;
+  gw = cluster.config.hostLinks.${config.networking.hostName}.ipfsGateway;
   ipfsPort = 110;
   nameservers = lib.unique config.networking.nameservers;
 in
@@ -13,8 +13,6 @@ in
     depot.nixosModules.ipfs
   ];
 
-  links.ipfsGateway.protocol = "http";
-
   networking.firewall = {
     allowedTCPPorts = [ ipfsPort 4001 ];
     allowedUDPPorts = [ ipfsPort 4001 ];
@@ -157,7 +155,9 @@ in
         "fc00::/7"
         "fe80::/10"
       ];
-      IPAddressAllow = nameservers;
+      IPAddressAllow = nameservers ++ [
+        cluster.config.vars.meshNet.cidr
+      ];
     };
     postStart = "chmod 660 /run/ipfs/ipfs-api.sock";
   };

cluster/services/ipfs/remote-api.nix

@@ -1,9 +1,9 @@
-{ config, depot, ... }:
+{ cluster, config, depot, ... }:
 with depot.lib.nginx;
 let
   inherit (depot.lib.meta) domain;
   cfg = config.services.ipfs;
-  gw = config.links.ipfsGateway;
+  gw = cluster.config.hostLinks.${config.networking.hostName}.ipfsGateway;
 in
 {
   users.users.nginx.extraGroups = [ cfg.group ];

cluster/services/monitoring/default.nix

@@ -85,5 +85,8 @@ in
     };
   };
 
-  dns.records.monitoring.consulService = "grafana";
+  ways.monitoring = {
+    consulService = "grafana";
+    extras.locations."/".proxyWebsockets = true;
+  };
 }

cluster/services/monitoring/grafana-ha.nix

@@ -8,8 +8,6 @@ let
 
   inherit (config.networking) hostName;
 
-  svc = cluster.config.services.monitoring;
-
   iniList = lib.concatStringsSep " ";
 
   login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}";
@@ -93,36 +91,15 @@ in
     };
   };
 
-  services.nginx = {
-    upstreams.grafana-ha.servers = lib.mapAttrs' (_: links: lib.nameValuePair links.grafana.tuple {}) (lib.getAttrs (svc.nodes.grafana) hostLinks);
-
-    virtualHosts."monitoring.${domain}" = lib.recursiveUpdate (depot.lib.nginx.vhosts.proxy "http://grafana-ha") {
-      locations."/".proxyWebsockets = true;
-    };
-  };
-
-  security.acme.certs."monitoring.${domain}" = {
-    dnsProvider = "exec";
-    webroot = lib.mkForce null;
-  };
-
   consul.services.grafana = {
     mode = "manual";
-    definition = rec {
+    definition = {
       name = "grafana";
-      address = depot.reflection.interfaces.primary.addrPublic;
+      address = hostLinks.${hostName}.grafana.ipv4;
-      port = 443;
+      port = hostLinks.${hostName}.grafana.port;
       checks = [
-        rec {
-          name = "Frontend";
-          id = "service:grafana:frontend";
-          interval = "30s";
-          http = "https://${address}/healthz";
-          tls_server_name = "monitoring.${domain}";
-          header.Host = lib.singleton tls_server_name;
-        }
         {
-          name = "Backend";
+          name = "Grafana";
           id = "service:grafana:backend";
           interval = "5s";
           http = "${hostLinks.${hostName}.grafana.url}/healthz";

cluster/services/storage/default.nix

@@ -88,9 +88,19 @@ in
     };
   };
 
-  dns.records.garage.consulService = "garage";
+  ways = {
-  dns.records."^[^_].+\\.web\\.garage" = {
+    garage = {
-    consulService = "garage-web";
+      consulService = "garage";
-    rewrite.type = "regex";
+      extras.extraConfig = ''
+        client_max_body_size 4G;
+      '';
+    };
+    "web.garage" = {
+      consulService = "garage-web";
+      wildcard = true;
+      extras.locations."/".extraConfig = ''
+        proxy_set_header Host "$1.${config.links.garageWeb.hostname}";
+      '';
+    };
   };
 }

cluster/services/storage/garage-gateway.nix

@@ -1,8 +1,8 @@
 { config, cluster, depot, lib, ... }:
 
 let
-  linkS3 = cluster.config.links.garageS3;
+  linkS3 = cluster.config.hostLinks.${config.networking.hostName}.garageS3;
-  linkWeb = cluster.config.links.garageWeb;
+  linkWeb = cluster.config.hostLinks.${config.networking.hostName}.garageWeb;
 in
 
 {
@@ -10,60 +10,14 @@ in
 
   services.garage.settings.admin.api_bind_addr = config.links.garageMetrics.tuple;
 
-  services.nginx.virtualHosts = {
-    ${linkS3.hostname} = depot.lib.nginx.vhosts.basic // {
-      locations = {
-        "/".proxyPass = cluster.config.hostLinks.${config.networking.hostName}.garageS3.url;
-
-        "= /health".proxyPass = config.links.garageMetrics.url;
-      };
-      extraConfig = "client_max_body_size 4G;";
-    };
-    "${linkWeb.hostname}" = depot.lib.nginx.vhosts.basic // {
-      serverName = "~^(.+)\.${lib.escapeRegex linkWeb.hostname}$";
-      enableACME = false;
-      useACMEHost = linkWeb.hostname;
-      locations = {
-        "/" = {
-          proxyPass = cluster.config.hostLinks.${config.networking.hostName}.garageWeb.url;
-          extraConfig = ''
-            proxy_set_header Host "$1.${linkWeb.hostname}";
-          '';
-        };
-
-        "= /.internal-api/garage/health" = {
-          proxyPass = "${config.links.garageMetrics.url}/health";
-        };
-      };
-    };
-  };
-  security.acme.certs = {
-    ${linkS3.hostname} = {
-      dnsProvider = "exec";
-      webroot = lib.mkForce null;
-    };
-    ${linkWeb.hostname} = {
-      domain = "*.${linkWeb.hostname}";
-      dnsProvider = "exec";
-      group = "nginx";
-    };
-  };
   consul.services = {
     garage = {
       mode = "external";
-      definition = rec {
+      definition = {
         name = "garage";
-        address = depot.reflection.interfaces.primary.addrPublic;
+        address = linkS3.ipv4;
         inherit (linkS3) port;
         checks = [
-          {
-            name = "Frontend";
-            id = "service:garage:frontend";
-            interval = "60s";
-            http = "https://${address}/health";
-            tls_server_name = linkS3.hostname;
-            header.Host = lib.singleton linkS3.hostname;
-          }
           {
             name = "Garage Node";
             id = "service:garage:node";
@@ -76,19 +30,11 @@ in
     garage-web = {
       mode = "external";
       unit = "garage";
-      definition = rec {
+      definition = {
         name = "garage-web";
-        address = depot.reflection.interfaces.primary.addrPublic;
+        address = linkWeb.ipv4;
         inherit (linkWeb) port;
         checks = [
-          {
-            name = "Frontend";
-            id = "service:garage-web:frontend";
-            interval = "60s";
-            http = "https://${address}/.internal-api/garage/health";
-            tls_server_name = "healthcheck.${linkWeb.hostname}";
-            header.Host = lib.singleton "healthcheck.${linkWeb.hostname}";
-          }
           {
             name = "Garage Node";
             id = "service:garage-web:node";

cluster/services/ways/default.nix

@@ -0,0 +1,16 @@
+{ config, lib, ... }:
+
+{
+  imports = [
+    ./options
+  ];
+
+  services.ways = {
+    nodes.host = config.services.websites.nodes.host;
+    nixos.host = ./host.nix;
+  };
+
+  dns.records = lib.mapAttrs'
+    (_: cfg: lib.nameValuePair cfg.dnsRecord.name ({ ... }: { imports = [ cfg.dnsRecord.value ]; }))
+    config.ways;
+}

cluster/services/ways/host.nix

@@ -0,0 +1,103 @@
+{ cluster, config, depot, lib, pkgs, ... }:
+
+let
+  externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;
+
+  consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
+
+  consulHttpAddr = "${config.services.consul.extraConfig.addresses.http or "127.0.0.1"}:${toString (config.services.consul.extraConfig.ports.http or 8500)}";
+in
+
+{
+  services.nginx = {
+    virtualHosts = lib.mapAttrs' (name: cfg: {
+      name = cfg.name;
+      value = { ... }: {
+        imports = [
+          cfg.extras
+          {
+            forceSSL = true;
+            enableACME = !cfg.internal && !cfg.wildcard;
+            useACMEHost = lib.mkMerge [
+              (lib.mkIf cfg.internal cfg.domainSuffixInternal)
+              (lib.mkIf cfg.wildcard "${name}.${cfg.domainSuffix}")
+            ];
+            locations = lib.mkMerge [
+              {
+                "/".proxyPass = cfg.target;
+                "${cfg.healthCheckPath}".extraConfig = "access_log off;";
+              }
+              {
+                "/.well-known/ways/internal-health-check" = {
+                  return = ''200 "INTERNAL_OK\n"'';
+                  extraConfig = "access_log off;";
+                };
+              }
+            ];
+          }
+        ];
+      };
+    }) cluster.config.ways;
+
+    appendHttpConfig = lib.mkIf (consulServiceWays != {}) ''
+      include /run/consul-template/nginx-ways-*.conf;
+    '';
+  };
+
+  security.acme.certs = lib.mapAttrs' (name: cfg: {
+    name = "${name}.${cfg.domainSuffix}";
+    value = {
+      domain = lib.mkIf cfg.wildcard "*.${name}.${cfg.domainSuffix}";
+      dnsProvider = "exec";
+      webroot = lib.mkForce null;
+      group = "nginx";
+    };
+  }) externalWays;
+
+  systemd.services = lib.mapAttrs' (name: cfg: {
+    name = "acme-${name}.${cfg.domainSuffix}";
+    value.distributed.enable = true;
+  }) externalWays;
+
+  services.consul-template.instances.ways = lib.mkIf (consulServiceWays != {}) {
+    user = "nginx";
+    group = "nginx";
+    settings = {
+      consul.address = "http://${consulHttpAddr}";
+      template = [
+        {
+          source = let
+            upstreams = lib.mapAttrsToList (_: cfg: ''
+              upstream ${cfg.nginxUpstreamName} {
+                {{ range $i, $e := service "${cfg.consulService}~_agent" -}}
+                server {{ .Address }}:{{ .Port }}{{ if ne $i 0 }} backup{{ end }};
+                {{end}}
+              }
+            '') consulServiceWays;
+          in pkgs.writeText "ways-upstreams.ctmpl" (lib.concatStringsSep "\n" (lib.unique upstreams));
+          destination = "/run/consul-template/nginx-ways-upstreams.conf";
+          exec.command = [
+            "${config.services.nginx.package}/bin/nginx"
+            "-s" "reload"
+            "-g" "pid /run/nginx/nginx.pid;"
+          ];
+        }
+      ];
+    };
+  };
+
+  consul.services.ways-proxy = {
+    unit = "nginx";
+    mode = "external";
+    definition = {
+      name = "ways-proxy";
+      address = depot.reflection.interfaces.primary.addrPublic;
+      port = 443;
+      checks = lib.singleton {
+        interval = "60s";
+        tcp = "127.0.0.1:80";
+      };
+      tags = lib.attrNames externalWays;
+    };
+  };
+}

cluster/services/ways/options/default.nix

@@ -0,0 +1,12 @@
+{ lib, depot, ... }:
+
+{
+  options.ways = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.submodule {
+      imports = [ ./way.nix ];
+      domainSuffixExternal = depot.lib.meta.domain;
+      domainSuffixInternal = "internal.${depot.lib.meta.domain}";
+    });
+    default = {};
+  };
+}

cluster/services/ways/options/way.nix

@@ -0,0 +1,97 @@
+{ config, lib, name, options, ... }:
+
+with lib;
+
+{
+  options = {
+    internal = mkOption {
+      description = "Whether to only make this Way available internally. Will use the internal subdomain.";
+      type = types.bool;
+      default = false;
+    };
+
+    name = mkOption {
+      description = "Domain name to use.";
+      type = types.str;
+      default = let
+        basename = "${name}.${config.domainSuffix}";
+      in if config.wildcard then "~^(.+)\.${lib.escapeRegex basename}$" else basename;
+    };
+
+    dnsRecord = {
+      name = mkOption {
+        description = "DNS record name for this Way.";
+        type = types.str;
+        default = if config.wildcard then "^[^_].+\\.${lib.escapeRegex name}" else name;
+      };
+
+      value = mkOption {
+        description = "DNS record value for this Way.";
+        type = types.deferredModule;
+        default = {
+          consulService = "${name}.ways-proxy";
+          rewrite.type = lib.mkIf config.wildcard "regex";
+        };
+      };
+    };
+
+    target = mkOption {
+      type = types.str;
+    };
+
+    wildcard = mkOption {
+      type = types.bool;
+      default = false;
+    };
+
+    consulService = mkOption {
+      type = types.str;
+    };
+
+    healthCheckPath = mkOption {
+      type = types.path;
+      default = "/.well-known/ways/internal-health-check";
+    };
+
+    useConsul = mkOption {
+      type = types.bool;
+      internal = true;
+      default = false;
+    };
+
+    nginxUpstreamName = mkOption {
+      type = types.str;
+      internal = true;
+    };
+
+    domainSuffixInternal = mkOption {
+      type = types.str;
+      internal = true;
+    };
+
+    domainSuffixExternal = mkOption {
+      type = types.str;
+      internal = true;
+    };
+
+    domainSuffix = mkOption {
+      type = types.str;
+      internal = true;
+      default = if config.internal then config.domainSuffixInternal else config.domainSuffixExternal;
+    };
+
+    extras = mkOption {
+      description = "Extra configuration to pass to the nginx virtual host submodule.";
+      type = types.deferredModule;
+      default = {};
+    };
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf options.consulService.isDefined {
+      useConsul = true;
+      nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
+      target = "http://${options.nginxUpstreamName.value}";
+    })
+  ];
+}