cluster/simulacrum: only run checks on x86_64-linux

cluster/secrets/attic-serverToken.age

@@ -1,11 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 NO562A mLMev+YA6zSxAWIIlwseZk8Skl4hfNNsWQPmLV7DxTo
-AEi55ZXzyYbZyludcP5Yywx7QDgFODh6z8+M2nxMAl4
--> ssh-ed25519 5/zT0w 91baPvXx4UdmyYCCIqc1M+Cb7pqdSx3/cgmfuexeUgY
-kePQp8flAsXPMLxJiQPoJuHEPPI+FzaSF+VL9U9jhwI
--> ssh-ed25519 d3WGuA U8Q68hN+5fI4xto/lpCiVyts00ezCzftfLvFFew7aiY
-B4wv05Y2gpl5qjV1Rbc6JSJk3coN6TFMB5FspwzLnlI
--> :0eX-grease
-ghW6iCUZj0e04I8Ba3CHzg
---- aHnzzTi1WxtHXGcQO1hNgmy04wyyObmYBcSq5fmbnAg
-Ñdï<EFBFBD>ÎÁŽ#¹<>¬sä®nƒŒó¤ž§–F#5IZ<49><5A>¯áË2wb®×¨âÑÞüË›oœkm÷ÒåN&"¤ü0LeÑzIjx—µzxF€´>ršúEq´Ý¤¥A‘nx¿šB!@‰ÕŸÆò2r©:ïm5í-Xl5çAåð*ëÌSV¿R3`Ð艨{ÿò<C3BF>ï©#ÍJgHÖ‡ÊÉg
+-> ssh-ed25519 d3WGuA ZLjCSe5wrN6abvvRmQjE+VXtRr+avP/CLPD7djXNr0M
+g8i9ambJGL2Q+ZLB6c6MxV9BryAgX4qZctJ9qByJ4n8
+-> ssh-ed25519 P/nEqQ zSGcZuufOAnTkPr74ZjwyISdLlfxBxqgmyWivxq1/Uo
+gArusBfIfsZ5/gwMYHLzDHTbgVGWDttbi0IAhvclRO4
+-> ssh-ed25519 YIaSKQ J4Fy0VSjdMPRgzysQptIUKiRR0TAgu0q1BYhtIpGkWU
+kKzmF3OUbGU40d33R15nMraUDZiFRoz9Z00XjjSk9Jw
+-> ssh-ed25519 NO562A BNQV8JodzTiNs/V+rFQxcsrhKJ3nRIFtWk6VxHzCRio
+ZyauAdOrPbADSDdBQoB+39MB2r7Ro4d0XwZIjf2z9Jo
+-> ssh-ed25519 5/zT0w hdMuyOmNKTlMKPn4w9VQFVXZkJNm1XSPAZ/Zip5WW04
+wcnur+BRQPqKzpV3vl7pn1VIGRK3GxQEUaQIefrZuI4
+--- 5AdxXgFmDm2w012QjpJ3gqlbfvkPm8fkEJjm8kV18G0
+&Ãf§äIT¼-ÿY!ŒÍ,Vu<56>Â9õÿöBFrœŠ´½4–ù™BÕÝ/®UäH˜rþ¸ž	#ƒˆç
ÄÝÕº†®UóQ¢ÿŽx$G{ÅŠMà2¡^/˜§¥Éè?12É¿t1©¿í¸&[}nêDAÛlýÑýˆ8uG®éZŽ×b¯èàîåd:@ÿ!Õþ
jîƒÚáÈNµrâlA³~

cluster/services/attic/binary-cache.nix

@@ -1,10 +1,16 @@
-{ config, cluster, depot, ... }:
+{ config, cluster, depot, lib, ... }:
 with depot.lib.nginx;
 {
-  links.garageNixStoreInternalRedirect = {
-    protocol = "http";
-    path = "/nix-store";
+  links = {
+    atticNixStoreInternalRedirect.protocol = "http";
+    garageNixStoreInternalRedirect.protocol = "http";
   };
+
+  security.acme.certs."cache.${depot.lib.meta.domain}" = {
+    dnsProvider = "exec";
+    webroot = lib.mkForce null;
+  };
+
   services.nginx.upstreams = {
       nar-serve.extraConfig = ''
       random;
@@ -12,10 +18,10 @@ with depot.lib.nginx;
       server ${config.links.nar-serve-nixos-org.tuple} fail_timeout=0;
     '';
     nix-store.servers = {
-      "${config.links.atticServer.tuple}" = {
+      "${config.links.garageNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
       };
-      "${config.links.garageNixStoreInternalRedirect.tuple}" = {
+      "${config.links.atticNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
       };
     };
@@ -28,7 +34,7 @@ with depot.lib.nginx;
       locations = {
         "= /".return = "302 /404";
         "/" = {
-          proxyPass = "http://nix-store/nix-store$request_uri";
+          proxyPass = "http://nix-store";
           extraConfig = ''
             proxy_next_upstream error http_500 http_502 http_404;
           '';
@@ -56,13 +62,29 @@ with depot.lib.nginx;
           inherit (config.links.garageNixStoreInternalRedirect) port;
         }
       ];
-      locations."~ ^${config.links.garageNixStoreInternalRedirect.path}/(.*)" = {
-        proxyPass = with cluster.config.links.garageWeb; "${protocol}://nix-store.${hostname}/$1";
+      locations."/" = {
+        proxyPass = with cluster.config.links.garageWeb; "${protocol}://nix-store.${hostname}";
         recommendedProxySettings = false;
         extraConfig = ''
           proxy_set_header Host "nix-store.${cluster.config.links.garageWeb.hostname}";
         '';
       };
     };
+    "attic-nix-store.internal.${depot.lib.meta.domain}" = {
+      serverName = "127.0.0.1";
+      listen = [
+        {
+          addr = "127.0.0.1";
+          inherit (config.links.atticNixStoreInternalRedirect) port;
+        }
+      ];
+      locations."/" = {
+        proxyPass = "https://cache-api.${depot.lib.meta.domain}/nix-store$request_uri";
+        recommendedProxySettings = false;
+        extraConfig = ''
+          proxy_set_header Host "cache-api.${depot.lib.meta.domain}";
+        '';
+      };
+    };
   };
 }

cluster/services/attic/default.nix

@@ -3,15 +3,23 @@
 {
   services.attic = {
     nodes = {
-      server = [ "VEGAS" ];
+      monolith = [ "VEGAS" "prophet" ];
+      server = [ "VEGAS" "grail" "prophet" ];
     };
     nixos = {
+      monolith = [
+        ./server.nix
+      ];
       server = [
         ./server.nix
         ./binary-cache.nix
         ./nar-serve.nix
       ];
     };
+    meshLinks.server = {
+      name = "attic";
+      link.protocol = "http";
+    };
     secrets = let
       inherit (config.services.attic) nodes;
     in {
@@ -41,7 +49,13 @@
       (node: depot.hours.${node}.interfaces.primary.addrPublic)
       config.services.attic.nodes.server;
   in config.lib.forService "attic" {
-    cache-api.target = serverAddrs;
     cache.target = serverAddrs;
   };
+
+  ways.cache-api = {
+    consulService = "atticd";
+    extras.extraConfig = ''
+      client_max_body_size 4G;
+    '';
+  };
 }

cluster/services/attic/server.nix

@@ -2,6 +2,10 @@
 
 let
   inherit (cluster.config.services.attic) secrets;
+
+  link = cluster.config.hostLinks.${config.networking.hostName}.attic;
+
+  isMonolith = lib.elem config.networking.hostName cluster.config.services.attic.nodes.monolith;
 in
 
 {
@@ -9,8 +13,6 @@ in
     depot.inputs.attic.nixosModules.atticd
   ];
 
-  links.atticServer.protocol = "http";
-
   services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];
 
   services.atticd = {
@@ -18,9 +20,10 @@ in
     package = depot.inputs.attic.packages.attic-server;
 
     credentialsFile = secrets.serverToken.path;
+    mode = if isMonolith then "monolithic" else "api-server";
 
     settings = {
-      listen = config.links.atticServer.tuple;
+      listen = link.tuple;
 
       chunking = {
         nar-size-threshold = 0;
@@ -59,8 +62,13 @@ in
 
   systemd.services.atticd = {
     after = [ "postgresql.service" ];
+    distributed = lib.mkIf isMonolith {
+      enable = true;
+      registerService = "atticd";
+    };
     serviceConfig = {
       DynamicUser = lib.mkForce false;
+      RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
     };
     environment = {
       AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
@@ -68,9 +76,20 @@ in
     };
   };
 
-  services.nginx.virtualHosts."cache-api.${depot.lib.meta.domain}" = depot.lib.nginx.vhosts.proxy config.links.atticServer.url // {
-    extraConfig = ''
-      client_max_body_size 4G;
-    '';
+  consul.services.atticd = {
+    mode = if isMonolith then "manual" else "direct";
+    definition = {
+      name = "atticd";
+      address = link.ipv4;
+      inherit (link) port;
+      checks = [
+        {
+          name = "Attic Server";
+          id = "service:atticd:backend";
+          interval = "5s";
+          http = link.url;
+        }
+      ];
+    };
   };
 }

cluster/services/monitoring/client.nix

@@ -26,7 +26,7 @@ in {
         name = "logging";
         positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
         clients = singleton {
-          url = "${cluster.config.links.loki-ingest.url}/loki/api/v1/push";
+          url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
         };
         scrape_configs = singleton {
           job_name = "journal";

cluster/services/monitoring/default.nix

@@ -14,14 +14,6 @@ in
   ];
 
   links = {
-    loki-ingest = {
-      protocol = "http";
-      ipv4 = meshIpFor "logging";
-    };
-    loki = {
-      protocol = "http";
-      ipv4 = meshIpFor "logging";
-    };
     prometheus-ingest = {
       protocol = "http";
       ipv4 = meshIpFor "server";
@@ -58,7 +50,7 @@ in
       client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
       blackbox = [ "checkmate" "grail" "prophet" ];
       grafana = [ "VEGAS" "prophet" ];
-      logging = [ "VEGAS" ];
+      logging = [ "VEGAS" "grail" ];
       server = [ "VEGAS" ];
     };
     nixos = {
@@ -74,21 +66,45 @@ in
         ./tracing.nix
       ];
     };
+    meshLinks.logging = {
+      name = "loki";
+      link.protocol = "http";
+    };
   };
 
   garage = {
     keys = {
-      loki = { };
+      loki-ingest.locksmith = {
+        nodes = config.services.monitoring.nodes.logging;
+        format = "envFile";
+      };
+      loki-query.locksmith = {
+        nodes = config.services.monitoring.nodes.logging;
+        format = "envFile";
+      };
       tempo = { };
     };
     buckets = {
-      loki-chunks.allow.loki = [ "read" "write" ];
+      loki-chunks.allow = {
+        loki-ingest = [ "read" "write" ];
+        loki-query = [ "read" ];
+      };
       tempo-chunks.allow.tempo = [ "read" "write" ];
     };
   };
 
-  ways.monitoring = {
-    consulService = "grafana";
-    extras.locations."/".proxyWebsockets = true;
+  ways = {
+    monitoring = {
+      consulService = "grafana";
+      extras.locations."/".proxyWebsockets = true;
+    };
+    monitoring-logs = {
+      internal = true;
+      consulService = "loki";
+      extras.extraConfig = ''
+        client_max_body_size 4G;
+        proxy_read_timeout 3600s;
+      '';
+    };
   };
 }

cluster/services/monitoring/grafana-ha.nix

@@ -2,7 +2,7 @@
 let
   inherit (depot.lib.meta) domain;
 
-  inherit (cluster.config.links) loki-ingest prometheus-ingest;
+  inherit (cluster.config.links) prometheus-ingest;
 
   inherit (cluster.config) hostLinks;
 
@@ -70,7 +70,7 @@ in
         {
           name = "Loki";
           uid = "P8E80F9AEF21F6940";
-          inherit (loki-ingest) url;
+          inherit (cluster.config.ways.monitoring-logs) url;
           type = "loki";
         }
       ];

cluster/services/monitoring/logging.nix

@@ -3,17 +3,25 @@
 let
   inherit (config.links) loki-grpc;
 
-  inherit (cluster.config.links) loki-ingest;
+  link = cluster.config.hostLinks.${config.networking.hostName}.loki;
 
   cfg = config.services.loki;
 in
 {
-  age.secrets.lokiSecrets.file = ./secrets/loki-secrets.age;
   links.loki-grpc.protocol = "grpc";
   systemd.services.loki = {
+    distributed = {
+      enable = true;
+      registerService = "loki";
+    };
     after = [ "wireguard-wgmesh.service" ];
-    serviceConfig.EnvironmentFile = config.age.secrets.lokiSecrets.path;
+    serviceConfig.EnvironmentFile = "/run/locksmith/garage-loki-ingest";
   };
+
+  services.locksmith.waitForSecrets.loki = [
+    "garage-loki-ingest"
+  ];
+
   services.loki = {
     enable = true;
     dataDir = "/srv/storage/private/loki";
@@ -22,8 +30,8 @@ in
       auth_enabled = false;
       server = {
         log_level = "warn";
-        http_listen_address = loki-ingest.ipv4;
-        http_listen_port = loki-ingest.port;
+        http_listen_address = link.ipv4;
+        http_listen_port = link.port;
         grpc_listen_address = loki-grpc.ipv4;
         grpc_listen_port = loki-grpc.port;
       };
@@ -104,4 +112,21 @@ in
       querier.max_concurrent = 16;
     };
   };
+
+  consul.services.loki = {
+    mode = "manual";
+    definition = {
+      name = "loki";
+      address = link.ipv4;
+      inherit (link) port;
+      checks = [
+        {
+          name = "Loki";
+          id = "service:loki:backend";
+          interval = "5s";
+          http = "${link.url}/ready";
+        }
+      ];
+    };
+  };
 }

cluster/services/nginx/nginx.nix

@@ -1,4 +1,4 @@
-{ depot, ... }:
+{ config, depot, ... }:
 
 let
   inherit (depot.lib.meta) adminEmail;
@@ -13,7 +13,7 @@ in {
     recommendedGzipSettings = true;
     proxyResolveWhileRunning = true;
     resolver = {
-      addresses = [ "127.0.0.1" ];
+      addresses = config.networking.nameservers;
       valid = "30s";
     };
     appendHttpConfig = ''

cluster/services/patroni/default.nix

@@ -9,7 +9,7 @@
   services.patroni = {
     nodes = {
       worker = [ "grail" "thunderskin" "VEGAS" ];
-      haproxy = [ "checkmate" "VEGAS" "prophet" ];
+      haproxy = [ "checkmate" "grail" "VEGAS" "prophet" ];
     };
     nixos = {
       worker = [

cluster/services/ways/default.nix

@@ -11,6 +11,9 @@
   };
 
   dns.records = lib.mapAttrs'
-    (_: cfg: lib.nameValuePair cfg.dnsRecord.name ({ ... }: { imports = [ cfg.dnsRecord.value ]; }))
+    (_: cfg: lib.nameValuePair cfg.dnsRecord.name ({ ... }: {
+      imports = [ cfg.dnsRecord.value ];
+      root = cfg.domainSuffix;
+    }))
     config.ways;
 }

cluster/services/ways/host.nix

@@ -3,6 +3,8 @@
 let
   externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;
 
+  internalWays = lib.filterAttrs (_: cfg: cfg.internal) cluster.config.ways;
+
   consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
 in
 
@@ -14,6 +16,7 @@ in
         imports = [
           cfg.extras
           {
+            listenAddresses = lib.mkIf cfg.internal [ config.reflection.interfaces.vstub.addr ];
             forceSSL = true;
             enableACME = !cfg.internal && !cfg.wildcard;
             useACMEHost = lib.mkMerge [
@@ -84,18 +87,34 @@ in
     };
   };
 
-  consul.services.ways-proxy = {
-    unit = "nginx";
-    mode = "external";
-    definition = {
-      name = "ways-proxy";
-      address = config.reflection.interfaces.primary.addrPublic;
-      port = 443;
-      checks = lib.singleton {
-        interval = "60s";
-        tcp = "127.0.0.1:80";
+  consul.services = {
+    ways-proxy = {
+      unit = "nginx";
+      mode = "external";
+      definition = {
+        name = "ways-proxy";
+        address = config.reflection.interfaces.primary.addrPublic;
+        port = 443;
+        checks = lib.singleton {
+          interval = "60s";
+          tcp = "127.0.0.1:80";
+        };
+        tags = lib.attrNames externalWays;
+      };
+    };
+    ways-proxy-internal = {
+      unit = "nginx";
+      mode = "external";
+      definition = {
+        name = "ways-proxy-internal";
+        address = config.reflection.interfaces.vstub.addr;
+        port = 443;
+        checks = lib.singleton {
+          interval = "60s";
+          tcp = "127.0.0.1:80";
+        };
+        tags = lib.attrNames internalWays;
       };
-      tags = lib.attrNames externalWays;
     };
   };
 }

cluster/services/ways/options/way.nix

@@ -29,7 +29,7 @@ with lib;
         description = "DNS record value for this Way.";
         type = types.deferredModule;
         default = {
-          consulService = "${name}.ways-proxy";
+          consulService = "${name}.${if config.internal then "ways-proxy-internal" else "ways-proxy"}";
           rewrite.type = lib.mkIf config.wildcard "regex";
         };
       };
@@ -57,6 +57,12 @@ with lib;
       default = "/.well-known/ways/internal-health-check";
     };
 
+    url = mkOption {
+      type = types.str;
+      readOnly = true;
+      default = "https://${name}.${config.domainSuffix}";
+    };
+
     useConsul = mkOption {
       type = types.bool;
       internal = true;

secrets.nix

@@ -9,7 +9,6 @@ in with hosts;
   "cluster/services/dns/acme-dns-db-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
   "cluster/services/monitoring/secrets/grafana-db-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "cluster/services/monitoring/secrets/grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
-  "cluster/services/monitoring/secrets/loki-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/monitoring/secrets/secret-monitoring/blackbox.age".publicKeys = max ++ map systemKeys [ checkmate grail prophet ];
   "cluster/services/monitoring/secrets/tempo-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/storage/secrets/heresy-encryption-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];