cluster/services/attic: make HA

cluster/secrets/attic-serverToken.age

@@ -1,11 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 NO562A mLMev+YA6zSxAWIIlwseZk8Skl4hfNNsWQPmLV7DxTo
-AEi55ZXzyYbZyludcP5Yywx7QDgFODh6z8+M2nxMAl4
--> ssh-ed25519 5/zT0w 91baPvXx4UdmyYCCIqc1M+Cb7pqdSx3/cgmfuexeUgY
-kePQp8flAsXPMLxJiQPoJuHEPPI+FzaSF+VL9U9jhwI
--> ssh-ed25519 d3WGuA U8Q68hN+5fI4xto/lpCiVyts00ezCzftfLvFFew7aiY
-B4wv05Y2gpl5qjV1Rbc6JSJk3coN6TFMB5FspwzLnlI
--> :0eX-grease
-ghW6iCUZj0e04I8Ba3CHzg
---- aHnzzTi1WxtHXGcQO1hNgmy04wyyObmYBcSq5fmbnAg
-Ñdï<EFBFBD>ÎÁŽ#¹<>¬sä®nƒŒó¤ž§–F#5IZ<49><5A>¯áË2wb®×¨âÑÞüË›oœkm÷ÒåN&"¤ü0LeÑzIjx—µzxF€´>ršúEq´Ý¤¥A‘nx¿šB!@‰ÕŸÆò2r©:ïm5í-Xl5çAåð*ëÌSV¿R3`Ð艨{ÿò<C3BF>ï©#ÍJgHÖ‡ÊÉg
+-> ssh-ed25519 d3WGuA ZLjCSe5wrN6abvvRmQjE+VXtRr+avP/CLPD7djXNr0M
+g8i9ambJGL2Q+ZLB6c6MxV9BryAgX4qZctJ9qByJ4n8
+-> ssh-ed25519 P/nEqQ zSGcZuufOAnTkPr74ZjwyISdLlfxBxqgmyWivxq1/Uo
+gArusBfIfsZ5/gwMYHLzDHTbgVGWDttbi0IAhvclRO4
+-> ssh-ed25519 YIaSKQ J4Fy0VSjdMPRgzysQptIUKiRR0TAgu0q1BYhtIpGkWU
+kKzmF3OUbGU40d33R15nMraUDZiFRoz9Z00XjjSk9Jw
+-> ssh-ed25519 NO562A BNQV8JodzTiNs/V+rFQxcsrhKJ3nRIFtWk6VxHzCRio
+ZyauAdOrPbADSDdBQoB+39MB2r7Ro4d0XwZIjf2z9Jo
+-> ssh-ed25519 5/zT0w hdMuyOmNKTlMKPn4w9VQFVXZkJNm1XSPAZ/Zip5WW04
+wcnur+BRQPqKzpV3vl7pn1VIGRK3GxQEUaQIefrZuI4
+--- 5AdxXgFmDm2w012QjpJ3gqlbfvkPm8fkEJjm8kV18G0
+&Ãf§äIT¼-ÿY!ŒÍ,Vu<56>Â9õÿöBFrœŠ´½4–ù™BÕÝ/®UäH˜rþ¸ž	#ƒˆç
ÄÝÕº†®UóQ¢ÿŽx$G{ÅŠMà2¡^/˜§¥Éè?12É¿t1©¿í¸&[}nêDAÛlýÑýˆ8uG®éZŽ×b¯èàîåd:@ÿ!Õþ
jîƒÚáÈNµrâlA³~

cluster/services/attic/binary-cache.nix

@@ -1,10 +1,16 @@
-{ config, cluster, depot, ... }:
+{ config, cluster, depot, lib, ... }:
 with depot.lib.nginx;
 {
-  links.garageNixStoreInternalRedirect = {
-    protocol = "http";
-    path = "/nix-store";
+  links = {
+    atticNixStoreInternalRedirect.protocol = "http";
+    garageNixStoreInternalRedirect.protocol = "http";
   };
+
+  security.acme.certs."cache.${depot.lib.meta.domain}" = {
+    dnsProvider = "exec";
+    webroot = lib.mkForce null;
+  };
+
   services.nginx.upstreams = {
       nar-serve.extraConfig = ''
       random;
@@ -12,10 +18,10 @@ with depot.lib.nginx;
       server ${config.links.nar-serve-nixos-org.tuple} fail_timeout=0;
     '';
     nix-store.servers = {
-      "${config.links.atticServer.tuple}" = {
+      "${config.links.garageNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
       };
-      "${config.links.garageNixStoreInternalRedirect.tuple}" = {
+      "${config.links.atticNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
       };
     };
@@ -28,7 +34,7 @@ with depot.lib.nginx;
       locations = {
         "= /".return = "302 /404";
         "/" = {
-          proxyPass = "http://nix-store/nix-store$request_uri";
+          proxyPass = "http://nix-store";
           extraConfig = ''
             proxy_next_upstream error http_500 http_502 http_404;
           '';
@@ -56,13 +62,29 @@ with depot.lib.nginx;
           inherit (config.links.garageNixStoreInternalRedirect) port;
         }
       ];
-      locations."~ ^${config.links.garageNixStoreInternalRedirect.path}/(.*)" = {
-        proxyPass = with cluster.config.links.garageWeb; "${protocol}://nix-store.${hostname}/$1";
+      locations."/" = {
+        proxyPass = with cluster.config.links.garageWeb; "${protocol}://nix-store.${hostname}";
         recommendedProxySettings = false;
         extraConfig = ''
           proxy_set_header Host "nix-store.${cluster.config.links.garageWeb.hostname}";
         '';
       };
     };
+    "attic-nix-store.internal.${depot.lib.meta.domain}" = {
+      serverName = "127.0.0.1";
+      listen = [
+        {
+          addr = "127.0.0.1";
+          inherit (config.links.atticNixStoreInternalRedirect) port;
+        }
+      ];
+      locations."/" = {
+        proxyPass = "https://cache-api.${depot.lib.meta.domain}/nix-store$request_uri";
+        recommendedProxySettings = false;
+        extraConfig = ''
+          proxy_set_header Host "cache-api.${depot.lib.meta.domain}";
+        '';
+      };
+    };
   };
 }

cluster/services/attic/default.nix

@@ -3,15 +3,23 @@
 {
   services.attic = {
     nodes = {
-      server = [ "VEGAS" ];
+      monolith = [ "VEGAS" "prophet" ];
+      server = [ "VEGAS" "grail" "prophet" ];
     };
     nixos = {
+      monolith = [
+        ./server.nix
+      ];
       server = [
         ./server.nix
         ./binary-cache.nix
         ./nar-serve.nix
       ];
     };
+    meshLinks.server = {
+      name = "attic";
+      link.protocol = "http";
+    };
     secrets = let
       inherit (config.services.attic) nodes;
     in {
@@ -41,7 +49,13 @@
       (node: depot.hours.${node}.interfaces.primary.addrPublic)
       config.services.attic.nodes.server;
   in {
-    cache-api.target = serverAddrs;
     cache.target = serverAddrs;
   };
+
+  ways.cache-api = {
+    consulService = "atticd";
+    extras.extraConfig = ''
+      client_max_body_size 4G;
+    '';
+  };
 }

cluster/services/attic/server.nix

@@ -2,6 +2,10 @@
 
 let
   inherit (cluster.config.services.attic) secrets;
+
+  link = cluster.config.hostLinks.${config.networking.hostName}.attic;
+
+  isMonolith = lib.elem config.networking.hostName cluster.config.services.attic.nodes.monolith;
 in
 
 {
@@ -9,8 +13,6 @@ in
     depot.inputs.attic.nixosModules.atticd
   ];
 
-  links.atticServer.protocol = "http";
-
   services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];
 
   services.atticd = {
@@ -18,9 +20,10 @@ in
     package = depot.inputs.attic.packages.attic-server;
 
     credentialsFile = secrets.serverToken.path;
+    mode = if isMonolith then "monolithic" else "api-server";
 
     settings = {
-      listen = config.links.atticServer.tuple;
+      listen = link.tuple;
 
       chunking = {
         nar-size-threshold = 0;
@@ -59,8 +62,13 @@ in
 
   systemd.services.atticd = {
     after = [ "postgresql.service" ];
+    distributed = lib.mkIf isMonolith {
+      enable = true;
+      registerService = "atticd";
+    };
     serviceConfig = {
       DynamicUser = lib.mkForce false;
+      RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
     };
     environment = {
       AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
@@ -68,9 +76,20 @@ in
     };
   };
 
-  services.nginx.virtualHosts."cache-api.${depot.lib.meta.domain}" = depot.lib.nginx.vhosts.proxy config.links.atticServer.url // {
-    extraConfig = ''
-      client_max_body_size 4G;
-    '';
+  consul.services.atticd = {
+    mode = if isMonolith then "manual" else "direct";
+    definition = {
+      name = "atticd";
+      address = link.ipv4;
+      inherit (link) port;
+      checks = [
+        {
+          name = "Attic Server";
+          id = "service:atticd:backend";
+          interval = "5s";
+          http = link.url;
+        }
+      ];
+    };
   };
 }