cluster/simulacrum: only run checks on x86_64-linux

cluster/catalog/secrets.nix

@@ -46,6 +46,7 @@ in
               };
             }) // (if secretConfig.shared then let
               secretFile = "${svcName}-${secretName}.age";
+              snakeoilFile = "${svcName}-${secretName}-snakeoil.txt";
             in {
               editSecret = {
                 description = "Edit this secret";
@@ -54,7 +55,14 @@ in
                   agenix -e '${secretFile}'
                 '';
               };
-            } else lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
+              editSnakeoil = {
+                description = "Edit this secret's snakeoil";
+                command = ''
+                  $EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
+                '';
+              };
+            } else lib.mkMerge [
+              (lib.mapAttrs' (name: lib.nameValuePair "editSecretInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
                 secretFile = "${svcName}-${secretName}-${node}.age";
               in {
                 description = "Edit this secret for '${node}'";
@@ -62,7 +70,16 @@ in
                   ${setupCommands secretFile [ node ]}
                   agenix -e '${secretFile}'
                 '';
-            })));
+              })))
+              (lib.mapAttrs' (name: lib.nameValuePair "editSnakeoilInstance-${name}") (lib.genAttrs secretConfig.nodes (node: let
+                snakeoilFile = "${svcName}-${secretName}-${node}-snakeoil.txt";
+              in {
+                description = "Edit this secret's snakeoil for '${node}'";
+                command = ''
+                  $EDITOR "$PRJ_ROOT/cluster/secrets"/'${snakeoilFile}'
+                '';
+              })))
+            ]);
           };
         }) svcConfig.secrets))
         lib.concatLists

cluster/default.nix

@@ -16,6 +16,8 @@ lib.evalModules {
     ./lib/port-magic-multi.nix
     ./lib/mesh.nix
     ./lib/secrets.nix
+    ./lib/testing.nix
+    ./lib/lib.nix
 
     ./import-services.nix
   ];

cluster/lib/inject-nixos-config.nix

@@ -2,9 +2,9 @@
 with lib;
 
 {
-  options.out.injectNixosConfig = mkOption {
-    description = "NixOS configuration to inject into the given host.";
-    type = with types; functionTo raw;
+  options.out = mkOption {
+    description = "Output functions.";
+    type = with types; lazyAttrsOf (functionTo raw);
     default = const [];
   };
 }

cluster/lib/lib.nix

@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+
+{
+  options.lib = {
+    forService = lib.mkOption {
+      description = "Enable these definitions for a particular service only.";
+      type = lib.types.functionTo lib.types.raw;
+      readOnly = true;
+      default = service: lib.mkIf (!config.simulacrum || lib.any (s: s == service) config.testConfig.activeServices);
+    };
+  };
+}

cluster/lib/service-module.nix

@@ -52,6 +52,24 @@ in
       }));
       default = {};
     };
+    simulacrum = {
+      enable = mkEnableOption "testing this service in the Simulacrum";
+      deps = mkOption {
+        description = "Other services to include.";
+        type = with types; listOf str;
+        default = [];
+      };
+      settings = mkOption {
+        description = "NixOS test configuration.";
+        type = types.deferredModule;
+        default = {};
+      };
+      augments = mkOption {
+        description = "Cluster augments (will be propagated).";
+        type = types.deferredModule;
+        default = {};
+      };
+    };
   };
   config.otherNodes = builtins.mapAttrs (const filterGroup) config.nodes;
 }

cluster/lib/services.nix

@@ -39,7 +39,11 @@ in
     default = {};
   };
 
-  config.out.injectNixosConfig = hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) config.services)) ++ [
+  config.out = {
+    injectNixosConfigForServices = services: hostName: (lib.flatten (lib.mapAttrsToList (getHostConfigurations hostName) (lib.getAttrs services config.services))) ++ [
       introspectionModule
     ];
+
+    injectNixosConfig = config.out.injectNixosConfigForServices (lib.attrNames config.services);
+  };
 }

cluster/lib/testing.nix

@@ -0,0 +1,15 @@
+{ lib, ... }:
+
+{
+  options = {
+    simulacrum = lib.mkOption {
+      description = "Whether we are in the Simulacrum.";
+      type = lib.types.bool;
+      default = false;
+    };
+    testConfig = lib.mkOption {
+      type = lib.types.attrs;
+      readOnly = true;
+    };
+  };
+}

cluster/part.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ./catalog
+    ./simulacrum/checks.nix
   ];
 
   options.cluster = lib.mkOption {

cluster/secrets/attic-serverToken.age

@@ -1,11 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 NO562A mLMev+YA6zSxAWIIlwseZk8Skl4hfNNsWQPmLV7DxTo
-AEi55ZXzyYbZyludcP5Yywx7QDgFODh6z8+M2nxMAl4
--> ssh-ed25519 5/zT0w 91baPvXx4UdmyYCCIqc1M+Cb7pqdSx3/cgmfuexeUgY
-kePQp8flAsXPMLxJiQPoJuHEPPI+FzaSF+VL9U9jhwI
--> ssh-ed25519 d3WGuA U8Q68hN+5fI4xto/lpCiVyts00ezCzftfLvFFew7aiY
-B4wv05Y2gpl5qjV1Rbc6JSJk3coN6TFMB5FspwzLnlI
--> :0eX-grease
-ghW6iCUZj0e04I8Ba3CHzg
---- aHnzzTi1WxtHXGcQO1hNgmy04wyyObmYBcSq5fmbnAg
-Ñdï<EFBFBD>ÎÁŽ#¹<>¬sä®nƒŒó¤ž§–F#5IZ<49><5A>¯áË2wb®×¨âÑÞüË›oœkm÷ÒåN&"¤ü0LeÑzIjx—µzxF€´>ršúEq´Ý¤¥A‘nx¿šB!@‰ÕŸÆò2r©:ïm5í-Xl5çAåð*ëÌSV¿R3`Ð艨{ÿò<C3BF>ï©#ÍJgHÖ‡ÊÉg
+-> ssh-ed25519 d3WGuA ZLjCSe5wrN6abvvRmQjE+VXtRr+avP/CLPD7djXNr0M
+g8i9ambJGL2Q+ZLB6c6MxV9BryAgX4qZctJ9qByJ4n8
+-> ssh-ed25519 P/nEqQ zSGcZuufOAnTkPr74ZjwyISdLlfxBxqgmyWivxq1/Uo
+gArusBfIfsZ5/gwMYHLzDHTbgVGWDttbi0IAhvclRO4
+-> ssh-ed25519 YIaSKQ J4Fy0VSjdMPRgzysQptIUKiRR0TAgu0q1BYhtIpGkWU
+kKzmF3OUbGU40d33R15nMraUDZiFRoz9Z00XjjSk9Jw
+-> ssh-ed25519 NO562A BNQV8JodzTiNs/V+rFQxcsrhKJ3nRIFtWk6VxHzCRio
+ZyauAdOrPbADSDdBQoB+39MB2r7Ro4d0XwZIjf2z9Jo
+-> ssh-ed25519 5/zT0w hdMuyOmNKTlMKPn4w9VQFVXZkJNm1XSPAZ/Zip5WW04
+wcnur+BRQPqKzpV3vl7pn1VIGRK3GxQEUaQIefrZuI4
+--- 5AdxXgFmDm2w012QjpJ3gqlbfvkPm8fkEJjm8kV18G0
+&Ãf§äIT¼-ÿY!ŒÍ,Vu<56>Â9õÿöBFrœŠ´½4–ù™BÕÝ/®UäH˜rþ¸ž	#ƒˆç
ÄÝÕº†®UóQ¢ÿŽx$G{ÅŠMà2¡^/˜§¥Éè?12É¿t1©¿í¸&[}nêDAÛlýÑýˆ8uG®éZŽ×b¯èàîåd:@ÿ!Õþ
jîƒÚáÈNµrâlA³~

cluster/services/attic/binary-cache.nix

@@ -1,10 +1,16 @@
-{ config, cluster, depot, ... }:
+{ config, cluster, depot, lib, ... }:
 with depot.lib.nginx;
 {
-  links.garageNixStoreInternalRedirect = {
-    protocol = "http";
-    path = "/nix-store";
+  links = {
+    atticNixStoreInternalRedirect.protocol = "http";
+    garageNixStoreInternalRedirect.protocol = "http";
   };
+
+  security.acme.certs."cache.${depot.lib.meta.domain}" = {
+    dnsProvider = "exec";
+    webroot = lib.mkForce null;
+  };
+
   services.nginx.upstreams = {
       nar-serve.extraConfig = ''
       random;
@@ -12,10 +18,10 @@ with depot.lib.nginx;
       server ${config.links.nar-serve-nixos-org.tuple} fail_timeout=0;
     '';
     nix-store.servers = {
-      "${config.links.atticServer.tuple}" = {
+      "${config.links.garageNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
       };
-      "${config.links.garageNixStoreInternalRedirect.tuple}" = {
+      "${config.links.atticNixStoreInternalRedirect.tuple}" = {
         fail_timeout = 0;
       };
     };
@@ -28,7 +34,7 @@ with depot.lib.nginx;
       locations = {
         "= /".return = "302 /404";
         "/" = {
-          proxyPass = "http://nix-store/nix-store$request_uri";
+          proxyPass = "http://nix-store";
           extraConfig = ''
             proxy_next_upstream error http_500 http_502 http_404;
           '';
@@ -56,13 +62,29 @@ with depot.lib.nginx;
           inherit (config.links.garageNixStoreInternalRedirect) port;
         }
       ];
-      locations."~ ^${config.links.garageNixStoreInternalRedirect.path}/(.*)" = {
-        proxyPass = with cluster.config.links.garageWeb; "${protocol}://nix-store.${hostname}/$1";
+      locations."/" = {
+        proxyPass = with cluster.config.links.garageWeb; "${protocol}://nix-store.${hostname}";
         recommendedProxySettings = false;
         extraConfig = ''
           proxy_set_header Host "nix-store.${cluster.config.links.garageWeb.hostname}";
         '';
       };
     };
+    "attic-nix-store.internal.${depot.lib.meta.domain}" = {
+      serverName = "127.0.0.1";
+      listen = [
+        {
+          addr = "127.0.0.1";
+          inherit (config.links.atticNixStoreInternalRedirect) port;
+        }
+      ];
+      locations."/" = {
+        proxyPass = "https://cache-api.${depot.lib.meta.domain}/nix-store$request_uri";
+        recommendedProxySettings = false;
+        extraConfig = ''
+          proxy_set_header Host "cache-api.${depot.lib.meta.domain}";
+        '';
+      };
+    };
   };
 }

cluster/services/attic/default.nix

@@ -3,15 +3,23 @@
 {
   services.attic = {
     nodes = {
-      server = [ "VEGAS" ];
+      monolith = [ "VEGAS" "prophet" ];
+      server = [ "VEGAS" "grail" "prophet" ];
     };
     nixos = {
+      monolith = [
+        ./server.nix
+      ];
       server = [
         ./server.nix
         ./binary-cache.nix
         ./nar-serve.nix
       ];
     };
+    meshLinks.server = {
+      name = "attic";
+      link.protocol = "http";
+    };
     secrets = let
       inherit (config.services.attic) nodes;
     in {
@@ -25,7 +33,7 @@
     };
   };
 
-  garage = {
+  garage = config.lib.forService "attic" {
     keys.attic.locksmith = {
       nodes = config.services.attic.nodes.server;
       owner = "atticd";
@@ -40,8 +48,14 @@
     serverAddrs = map
       (node: depot.hours.${node}.interfaces.primary.addrPublic)
       config.services.attic.nodes.server;
-  in {
-    cache-api.target = serverAddrs;
+  in config.lib.forService "attic" {
     cache.target = serverAddrs;
   };
+
+  ways.cache-api = {
+    consulService = "atticd";
+    extras.extraConfig = ''
+      client_max_body_size 4G;
+    '';
+  };
 }

cluster/services/attic/server.nix

@@ -2,6 +2,10 @@
 
 let
   inherit (cluster.config.services.attic) secrets;
+
+  link = cluster.config.hostLinks.${config.networking.hostName}.attic;
+
+  isMonolith = lib.elem config.networking.hostName cluster.config.services.attic.nodes.monolith;
 in
 
 {
@@ -9,8 +13,6 @@ in
     depot.inputs.attic.nixosModules.atticd
   ];
 
-  links.atticServer.protocol = "http";
-
   services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];
 
   services.atticd = {
@@ -18,9 +20,10 @@ in
     package = depot.inputs.attic.packages.attic-server;
 
     credentialsFile = secrets.serverToken.path;
+    mode = if isMonolith then "monolithic" else "api-server";
 
     settings = {
-      listen = config.links.atticServer.tuple;
+      listen = link.tuple;
 
       chunking = {
         nar-size-threshold = 0;
@@ -59,8 +62,13 @@ in
 
   systemd.services.atticd = {
     after = [ "postgresql.service" ];
+    distributed = lib.mkIf isMonolith {
+      enable = true;
+      registerService = "atticd";
+    };
     serviceConfig = {
       DynamicUser = lib.mkForce false;
+      RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
     };
     environment = {
       AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
@@ -68,9 +76,20 @@ in
     };
   };
 
-  services.nginx.virtualHosts."cache-api.${depot.lib.meta.domain}" = depot.lib.nginx.vhosts.proxy config.links.atticServer.url // {
-    extraConfig = ''
-      client_max_body_size 4G;
-    '';
+  consul.services.atticd = {
+    mode = if isMonolith then "manual" else "direct";
+    definition = {
+      name = "atticd";
+      address = link.ipv4;
+      inherit (link) port;
+      checks = [
+        {
+          name = "Attic Server";
+          id = "service:atticd:backend";
+          interval = "5s";
+          http = link.url;
+        }
+      ];
+    };
   };
 }

cluster/services/consul/default.nix

@@ -22,6 +22,11 @@ in
       ];
       ready = ./ready.nix;
     };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" ];
+      settings = ./test.nix;
+    };
   };
 
   dns.records."consul-remote.internal".consulService = "consul-remote";

cluster/services/consul/test.nix

@@ -0,0 +1,19 @@
+{
+  testScript = ''
+    import json
+
+    start_all()
+
+    with subtest("should form cluster"):
+      for machine in machines:
+        machine.succeed("systemctl start consul-ready.service")
+      for machine in machines:
+        consulConfig = json.loads(machine.succeed("cat /etc/consul.json"))
+        addr = consulConfig["addresses"]["http"]
+        port = consulConfig["ports"]["http"]
+        setEnv = f"CONSUL_HTTP_ADDR={addr}:{port}"
+        memberList = machine.succeed(f"{setEnv} consul members --status=alive")
+        for machine2 in machines:
+          assert machine2.name in memberList
+  '';
+}

cluster/services/forge/default.nix

@@ -19,19 +19,21 @@
 
   ways.forge.target = let
     host = builtins.head config.services.forge.nodes.server;
-  in config.hostLinks.${host}.forge.url;
+  in config.lib.forService "forge" config.hostLinks.${host}.forge.url;
 
-  garage = {
+  garage = config.lib.forService "forge" {
     keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
     buckets.forgejo.allow.forgejo = [ "read" "write" ];
   };
 
-  monitoring.blackbox.targets.forge = {
+  monitoring.blackbox.targets.forge = config.lib.forService "forge" {
     address = "https://forge.${depot.lib.meta.domain}/api/v1/version";
     module = "https2xx";
   };
 
-  dns.records."ssh.forge".target = map
+  dns.records = config.lib.forService "forge" {
+    "ssh.forge".target = map
       (node: depot.hours.${node}.interfaces.primary.addrPublic)
       config.services.forge.nodes.server;
+  };
 }

cluster/services/frangiclave/default.nix

@@ -0,0 +1,27 @@
+{ config, ... }:
+
+{
+  services.frangiclave = {
+    nodes = {
+      server = [ "VEGAS" "grail" "prophet" ];
+      cluster = config.services.frangiclave.nodes.server;
+      agent = []; # all nodes, for vault-agent, secret templates, etc.
+    };
+    meshLinks = {
+      server.link.protocol = "http";
+      cluster.link.protocol = "http";
+    };
+    nixos = {
+      server = [
+        ./server.nix
+      ];
+      cluster = [];
+      agent = [];
+    };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" "consul" ];
+      settings = ./test.nix;
+    };
+  };
+}

cluster/services/frangiclave/server.nix

@@ -0,0 +1,34 @@
+{ cluster, config, depot, lib, ... }:
+
+let
+  apiLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server;
+  clusterLink = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-cluster;
+in
+
+{
+  services.vault = {
+    enable = true;
+    package = depot.packages.openbao;
+    address = apiLink.tuple;
+    extraConfig = /*hcl*/ ''
+      api_addr = "${apiLink.url}"
+      cluster_addr = "${clusterLink.url}"
+    '';
+
+    storageBackend = "raft";
+    storageConfig = /*hcl*/ ''
+      node_id = "x${builtins.hashString "sha256" "frangiclave-node-${config.networking.hostName}"}"
+      ${
+        lib.pipe (cluster.config.services.frangiclave.otherNodes.server config.networking.hostName) [
+          (map (node: cluster.config.hostLinks.${node}.frangiclave-server))
+          (map (link: /*hcl*/ ''
+            retry_join {
+              leader_api_addr = "${link.url}"
+            }
+          ''))
+          (lib.concatStringsSep "\n")
+        ]
+      }
+    '';
+  };
+}

cluster/services/frangiclave/test.nix

@@ -0,0 +1,12 @@
+{ lib, ... }:
+
+{
+  interactive.defaults = { cluster, config, ... }: {
+    config = lib.mkIf config.services.vault.enable {
+      environment.variables.VAULT_ADDR = cluster.config.hostLinks.${config.networking.hostName}.frangiclave-server.url;
+      environment.systemPackages = [ config.services.vault.package ];
+    };
+  };
+
+  testScript = "assert False";
+}

cluster/services/monitoring/client.nix

@@ -26,7 +26,7 @@ in {
         name = "logging";
         positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
         clients = singleton {
-          url = "${cluster.config.links.loki-ingest.url}/loki/api/v1/push";
+          url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
         };
         scrape_configs = singleton {
           job_name = "journal";

cluster/services/monitoring/default.nix

@@ -14,14 +14,6 @@ in
   ];
 
   links = {
-    loki-ingest = {
-      protocol = "http";
-      ipv4 = meshIpFor "logging";
-    };
-    loki = {
-      protocol = "http";
-      ipv4 = meshIpFor "logging";
-    };
     prometheus-ingest = {
       protocol = "http";
       ipv4 = meshIpFor "server";
@@ -58,7 +50,7 @@ in
       client = [ "checkmate" "grail" "thunderskin" "VEGAS" "prophet" ];
       blackbox = [ "checkmate" "grail" "prophet" ];
       grafana = [ "VEGAS" "prophet" ];
-      logging = [ "VEGAS" ];
+      logging = [ "VEGAS" "grail" ];
       server = [ "VEGAS" ];
     };
     nixos = {
@@ -74,21 +66,45 @@ in
         ./tracing.nix
       ];
     };
+    meshLinks.logging = {
+      name = "loki";
+      link.protocol = "http";
+    };
   };
 
   garage = {
     keys = {
-      loki = { };
+      loki-ingest.locksmith = {
+        nodes = config.services.monitoring.nodes.logging;
+        format = "envFile";
+      };
+      loki-query.locksmith = {
+        nodes = config.services.monitoring.nodes.logging;
+        format = "envFile";
+      };
       tempo = { };
     };
     buckets = {
-      loki-chunks.allow.loki = [ "read" "write" ];
+      loki-chunks.allow = {
+        loki-ingest = [ "read" "write" ];
+        loki-query = [ "read" ];
+      };
       tempo-chunks.allow.tempo = [ "read" "write" ];
     };
   };
 
-  ways.monitoring = {
+  ways = {
+    monitoring = {
       consulService = "grafana";
       extras.locations."/".proxyWebsockets = true;
     };
+    monitoring-logs = {
+      internal = true;
+      consulService = "loki";
+      extras.extraConfig = ''
+        client_max_body_size 4G;
+        proxy_read_timeout 3600s;
+      '';
+    };
+  };
 }

cluster/services/monitoring/grafana-ha.nix

@@ -2,7 +2,7 @@
 let
   inherit (depot.lib.meta) domain;
 
-  inherit (cluster.config.links) loki-ingest prometheus-ingest;
+  inherit (cluster.config.links) prometheus-ingest;
 
   inherit (cluster.config) hostLinks;
 
@@ -70,7 +70,7 @@ in
         {
           name = "Loki";
           uid = "P8E80F9AEF21F6940";
-          inherit (loki-ingest) url;
+          inherit (cluster.config.ways.monitoring-logs) url;
           type = "loki";
         }
       ];

cluster/services/monitoring/logging.nix

@@ -3,17 +3,25 @@
 let
   inherit (config.links) loki-grpc;
 
-  inherit (cluster.config.links) loki-ingest;
+  link = cluster.config.hostLinks.${config.networking.hostName}.loki;
 
   cfg = config.services.loki;
 in
 {
-  age.secrets.lokiSecrets.file = ./secrets/loki-secrets.age;
   links.loki-grpc.protocol = "grpc";
   systemd.services.loki = {
-    after = [ "wireguard-wgmesh.service" ];
-    serviceConfig.EnvironmentFile = config.age.secrets.lokiSecrets.path;
+    distributed = {
+      enable = true;
+      registerService = "loki";
     };
+    after = [ "wireguard-wgmesh.service" ];
+    serviceConfig.EnvironmentFile = "/run/locksmith/garage-loki-ingest";
+  };
+
+  services.locksmith.waitForSecrets.loki = [
+    "garage-loki-ingest"
+  ];
+
   services.loki = {
     enable = true;
     dataDir = "/srv/storage/private/loki";
@@ -22,8 +30,8 @@ in
       auth_enabled = false;
       server = {
         log_level = "warn";
-        http_listen_address = loki-ingest.ipv4;
-        http_listen_port = loki-ingest.port;
+        http_listen_address = link.ipv4;
+        http_listen_port = link.port;
         grpc_listen_address = loki-grpc.ipv4;
         grpc_listen_port = loki-grpc.port;
       };
@@ -104,4 +112,21 @@ in
       querier.max_concurrent = 16;
     };
   };
+
+  consul.services.loki = {
+    mode = "manual";
+    definition = {
+      name = "loki";
+      address = link.ipv4;
+      inherit (link) port;
+      checks = [
+        {
+          name = "Loki";
+          id = "service:loki:backend";
+          interval = "5s";
+          http = "${link.url}/ready";
+        }
+      ];
+    };
+  };
 }

cluster/services/nginx/nginx.nix

@@ -1,4 +1,4 @@
-{ depot, ... }:
+{ config, depot, ... }:
 
 let
   inherit (depot.lib.meta) adminEmail;
@@ -13,7 +13,7 @@ in {
     recommendedGzipSettings = true;
     proxyResolveWhileRunning = true;
     resolver = {
-      addresses = [ "127.0.0.1" ];
+      addresses = config.networking.nameservers;
       valid = "30s";
     };
     appendHttpConfig = ''

cluster/services/patroni/default.nix

@@ -9,7 +9,7 @@
   services.patroni = {
     nodes = {
       worker = [ "grail" "thunderskin" "VEGAS" ];
-      haproxy = [ "checkmate" "VEGAS" "prophet" ];
+      haproxy = [ "checkmate" "grail" "VEGAS" "prophet" ];
     };
     nixos = {
       worker = [

cluster/services/storage/default.nix

@@ -35,6 +35,8 @@ in
         ./garage.nix
         ./garage-options.nix
         ./garage-layout.nix
+      ] ++ lib.optionals config.simulacrum [
+        ./simulacrum/snakeoil-rpc-secret.nix
       ];
       garageConfig = [
         ./garage-gateway.nix
@@ -48,6 +50,11 @@ in
       garageInternal = [ ./garage-internal.nix ];
       garageExternal = [ ./garage-external.nix ];
     };
+    simulacrum = {
+      enable = true;
+      deps = [ "wireguard" "consul" "locksmith" ];
+      settings = ./test.nix;
+    };
   };
 
   links = {

cluster/services/storage/garage.nix

@@ -63,6 +63,8 @@ in
   };
 
   systemd.services.garage = {
+    requires = [ "consul-ready.service" ];
+    after = [ "consul-ready.service" ];
     unitConfig = {
       RequiresMountsFor = [ cfg.settings.data_dir ];
     };

cluster/services/storage/simulacrum/snakeoil-rpc-secret.nix

@@ -0,0 +1,3 @@
+{
+  environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+}

cluster/services/storage/test.nix

@@ -0,0 +1,106 @@
+{ cluster, lib, ... }:
+
+let
+  inherit (cluster.config.services.storage) nodes;
+
+  firstGarageNode = lib.elemAt nodes.garage 0;
+in
+
+{
+  nodes = lib.genAttrs nodes.garage (node: {
+    services.garage = {
+      layout.initial = lib.genAttrs nodes.garage (_: {
+        capacity = lib.mkOverride 51 1000;
+      });
+    };
+    specialisation.modifiedLayout = {
+      inheritParentConfig = true;
+      configuration = {
+        services.garage = {
+          layout.initial.${firstGarageNode}.capacity = lib.mkForce 2000;
+          keys.testKey.allow.createBucket = true;
+          buckets = {
+            bucket1 = {
+              allow.testKey = [ "read" "write" ];
+              quotas = {
+                maxObjects = 300;
+                maxSize = 400 * 1024 * 1024;
+              };
+            };
+            bucket2 = {
+              allow.testKey = [ "read" ];
+            };
+          };
+        };
+        system.ascensions.garage-layout.incantations = lib.mkForce (i: [
+          (i.runGarage ''
+            garage layout assign -z eu-central -c 2000 "$(garage node id -q | cut -d@ -f1)"
+            garage layout apply --version 2
+          '')
+        ]);
+      };
+    };
+  });
+
+  testScript = ''
+    import json
+    nodes = [n for n in machines if n.name in json.loads('${builtins.toJSON nodes.garage}')]
+    garage1 = nodes[0]
+
+    start_all()
+
+    with subtest("should bootstrap new cluster"):
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+    with subtest("should apply new layout with ascension"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
+
+      for node in nodes:
+        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should apply new layout from scratch"):
+      for node in nodes:
+        node.systemctl("stop garage.service")
+        node.succeed("rm -rf /var/lib/garage-metadata")
+
+      for node in nodes:
+        node.systemctl("start garage.service")
+
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should create specified buckets and keys"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
+      garage1.succeed("garage key list | grep testKey")
+      garage1.succeed("garage bucket list | grep bucket1")
+      garage1.succeed("garage bucket list | grep bucket2")
+
+    with subtest("should delete unspecified buckets and keys"):
+      garage1.succeed("garage bucket create unwantedbucket")
+      garage1.succeed("garage key new --name unwantedkey")
+      garage1.succeed("systemctl restart garage-apply.service")
+
+      garage1.fail("garage key list | grep unwantedkey")
+      garage1.fail("garage bucket list | grep unwantedbucket")
+  '';
+}
+

cluster/services/ways/default.nix

@@ -11,6 +11,9 @@
   };
 
   dns.records = lib.mapAttrs'
-    (_: cfg: lib.nameValuePair cfg.dnsRecord.name ({ ... }: { imports = [ cfg.dnsRecord.value ]; }))
+    (_: cfg: lib.nameValuePair cfg.dnsRecord.name ({ ... }: {
+      imports = [ cfg.dnsRecord.value ];
+      root = cfg.domainSuffix;
+    }))
     config.ways;
 }

cluster/services/ways/host.nix

@@ -3,6 +3,8 @@
 let
   externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;
 
+  internalWays = lib.filterAttrs (_: cfg: cfg.internal) cluster.config.ways;
+
   consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
 in
 
@@ -14,6 +16,7 @@ in
         imports = [
           cfg.extras
           {
+            listenAddresses = lib.mkIf cfg.internal [ config.reflection.interfaces.vstub.addr ];
             forceSSL = true;
             enableACME = !cfg.internal && !cfg.wildcard;
             useACMEHost = lib.mkMerge [
@@ -84,7 +87,8 @@ in
     };
   };
 
-  consul.services.ways-proxy = {
+  consul.services = {
+    ways-proxy = {
       unit = "nginx";
       mode = "external";
       definition = {
@@ -98,4 +102,19 @@ in
         tags = lib.attrNames externalWays;
       };
     };
+    ways-proxy-internal = {
+      unit = "nginx";
+      mode = "external";
+      definition = {
+        name = "ways-proxy-internal";
+        address = config.reflection.interfaces.vstub.addr;
+        port = 443;
+        checks = lib.singleton {
+          interval = "60s";
+          tcp = "127.0.0.1:80";
+        };
+        tags = lib.attrNames internalWays;
+      };
+    };
+  };
 }

cluster/services/ways/options/way.nix

@@ -29,7 +29,7 @@ with lib;
         description = "DNS record value for this Way.";
         type = types.deferredModule;
         default = {
-          consulService = "${name}.ways-proxy";
+          consulService = "${name}.${if config.internal then "ways-proxy-internal" else "ways-proxy"}";
           rewrite.type = lib.mkIf config.wildcard "regex";
         };
       };
@@ -57,6 +57,12 @@ with lib;
       default = "/.well-known/ways/internal-health-check";
     };
 
+    url = mkOption {
+      type = types.str;
+      readOnly = true;
+      default = "https://${name}.${config.domainSuffix}";
+    };
+
     useConsul = mkOption {
       type = types.bool;
       internal = true;

cluster/services/wireguard/default.nix

@@ -10,6 +10,19 @@ let
   };
 
   getExtAddr = host: host.interfaces.primary.addrPublic;
+
+  snakeoilPublicKeys = {
+    checkmate = "TESTtbFybW5YREwtd18a1A4StS4YAIUS5/M1Lv0jHjA=";
+    grail = "TEsTh7bthkaDh9A1CpqDi/F121ao5lRZqIJznLH8mB4=";
+    thunderskin = "tEST6afFmVN18o+EiWNFx+ax3MJwdQIeNfJSGEpffXw=";
+    VEGAS = "tEsT6s7VtM5C20eJBaq6UlQydAha8ATlmrTRe9T5jnM=";
+    prophet = "TEstYyb5IoqSL53HbSQwMhTaR16sxcWcMmXIBPd+1gE=";
+  };
+
+  grease = hourName: realPublicKey: if config.simulacrum then
+    snakeoilPublicKeys.${hourName}
+  else
+    realPublicKey;
 in
 {
   vars = {
@@ -22,7 +35,7 @@ in
       extra = {
         meshIp = "10.1.1.32";
         inherit meshNet;
-        pubKey = "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
+        pubKey = grease "checkmate" "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
         extraRoutes = [];
       };
     };
@@ -31,7 +44,7 @@ in
       extra = {
         meshIp = "10.1.1.6";
         inherit meshNet;
-        pubKey = "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
+        pubKey = grease "grail" "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
         extraRoutes = [];
       };
     };
@@ -40,7 +53,7 @@ in
       extra = {
         meshIp = "10.1.1.4";
         inherit meshNet;
-        pubKey = "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
+        pubKey = grease "thunderskin" "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
         extraRoutes = [];
       };
     };
@@ -49,7 +62,7 @@ in
       extra = {
         meshIp = "10.1.1.5";
         inherit meshNet;
-        pubKey = "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
+        pubKey = grease "VEGAS" "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
         extraRoutes = [ "${hours.VEGAS.interfaces.vstub.addr}/32" "10.10.0.0/16" ];
       };
     };
@@ -58,7 +71,7 @@ in
       extra = {
         meshIp = "10.1.1.9";
         inherit meshNet;
-        pubKey = "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
+        pubKey = grease "prophet" "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
         extraRoutes = [];
       };
     };
@@ -69,8 +82,12 @@ in
       storm = [ "VEGAS" ];
     };
     nixos = {
-      mesh = ./mesh.nix;
-      storm = ./storm.nix;
+      mesh = [
+        ./mesh.nix
+      ] ++ lib.optionals config.simulacrum [
+        ./simulacrum/snakeoil-keys.nix
+      ];
+      storm = [ ./storm.nix ];
     };
     secrets.meshPrivateKey = {
       nodes = config.services.wireguard.nodes.mesh;

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-VEGAS

@@ -0,0 +1 @@
+MNvWpMluuzQvPyGTp7jtyPSyz6n9lIly/WX1gW2NAHg=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-checkmate

@@ -0,0 +1 @@
+YHzP8rBP6qiXs6ZdnvHop9KnCYRADIEejwZzAzvj8m4=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-grail

@@ -0,0 +1 @@
+uD7X5E6N9d0sN+xPr/bWnehSa3bAok741GO7Z4I+Z3I=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-prophet

@@ -0,0 +1 @@
+QHyIJ3HoKGGFN28qOrQP4UyoQMP5bM7Idn2MzayKzEM=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-thunderskin

@@ -0,0 +1 @@
+YLl+hkWaCWx/5PpWs3cQ+bKqYdJef/qZ+FMTsM9ammM=

cluster/services/wireguard/simulacrum/snakeoil-keys.nix

@@ -0,0 +1,6 @@
+{ lib, config, ... }: {
+  config.environment.etc = {
+    "dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
+    "dummy-secrets/wireguard-key-storm".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
+  };
+}

cluster/simulacrum/checks.nix

@@ -0,0 +1,16 @@
+{ config, extendModules, lib, ... }:
+
+{
+  perSystem = { pkgs, system, ... }: {
+    checks = lib.mkIf (system == "x86_64-linux") (lib.mapAttrs' (name: svc: let
+      runSimulacrum = pkgs.callPackage ./. {
+        inherit config extendModules;
+      };
+    in {
+      name = "simulacrum-${name}";
+      value = runSimulacrum {
+        service = name;
+      };
+    }) (lib.filterAttrs (_: svc: svc.simulacrum.enable) config.cluster.config.services));
+  };
+}

cluster/simulacrum/default.nix

@@ -0,0 +1,117 @@
+{ testers, config, extendModules, lib, system }:
+
+{ service }:
+
+let
+  serviceConfig = config.cluster.config.services.${service};
+  serviceList = [ service ] ++ serviceConfig.simulacrum.deps;
+  allAugments = map (svc: config.cluster.config.services.${svc}.simulacrum.augments) serviceList;
+
+  lift = config;
+
+  snakeoil = {
+    ssh = {
+      public = lib.fileContents ../../packages/checks/snakeoil/ssh/snakeoil-key.pub;
+      private = ../../packages/checks/snakeoil/ssh/snakeoil-key;
+    };
+  };
+
+  nodes = lib.attrNames config.gods.fromLight;
+  digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes (lib.range 1 255));
+  depot' = extendModules {
+    modules = [
+      ({ config, ... }: {
+        gods.fromLight = lib.mapAttrs (name: cfg: {
+          interfaces.primary = {
+            link = lib.mkForce "vprimary";
+          };
+          ssh.id.publicKey = lib.mkForce snakeoil.ssh.public;
+        }) lift.gods.fromLight;
+
+        cluster = lib.mkForce (lift.cluster.extendModules {
+          specialArgs.depot = config;
+          modules = [
+            {
+              simulacrum = true;
+              testConfig = {
+                subject = service;
+                activeServices = serviceList;
+              };
+            }
+          ];
+        });
+      })
+    ];
+  };
+  specialArgs = depot'.config.lib.summon system lib.id;
+in
+
+testers.runNixOSTest {
+  name = "simulacrum-${service}";
+
+  imports = [
+    serviceConfig.simulacrum.settings
+  ] ++ allAugments;
+
+  _module.args = {
+    inherit (depot'.config) cluster;
+  };
+
+  node = { inherit specialArgs; };
+  nodes = lib.genAttrs nodes (node: let
+    hour = depot'.config.hours.${node};
+  in {
+    imports = [
+      specialArgs.depot.hours.${node}.nixos
+      ../../packages/checks/modules/nixos/age-dummy-secrets
+      ../../packages/checks/modules/nixos/external-storage.nix
+    ] ++ depot'.config.cluster.config.out.injectNixosConfigForServices serviceList node;
+
+    boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
+    networking = {
+      interfaces = {
+        ${hour.interfaces.primary.link} = {
+          useDHCP = lib.mkForce false;
+          virtual = true;
+          ipv4.addresses = lib.mkForce [
+            {
+              address = hour.interfaces.primary.addr;
+              prefixLength = 32;
+            }
+          ];
+        };
+        eth1.ipv4.routes = lib.pipe nodes [
+          (lib.filter (n: n != node))
+          (map (n: let
+            hour = depot'.config.hours.${n};
+          in {
+            address = hour.interfaces.primary.addrPublic;
+            prefixLength = 32;
+            via = "192.168.1.${toString digits.${n}}";
+          }))
+        ];
+      };
+
+      firewall.extraCommands = lib.mkAfter (lib.optionalString (hour.interfaces.primary.isNat) ''
+        # self-nat
+        iptables -t nat -A PREROUTING -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
+        iptables -t nat -A POSTROUTING -s ${hour.interfaces.primary.addr} -j SNAT --to-source ${hour.interfaces.primary.addrPublic}
+      '');
+    };
+
+    systemd.services = {
+      hyprspace.enable = false;
+    };
+
+    environment.etc = {
+      "ssh/ssh_host_ed25519_key" = {
+        source = snakeoil.ssh.private;
+        mode = "0400";
+      };
+    };
+    virtualisation = {
+      cores = 2;
+      memorySize = 4096;
+    };
+  });
+}

packages/catalog/checks.nix

@@ -1,10 +1,19 @@
 { lib, ... }:
 
 {
-  perSystem = { config, ... }: {
-    catalog.depot = {
-      checks = lib.mapAttrs (name: check: {
-        description = "NixOS Test: ${name}";
+  perSystem = { config, pkgs, ... }: {
+    catalog = lib.mkMerge (lib.mapAttrsToList (name': check: let
+      simulacrum = lib.hasPrefix "simulacrum-" name';
+      name = lib.removePrefix "simulacrum-" name';
+      baseAttrPath = if simulacrum then
+        [ "cluster" "simulacrum" ]
+      else
+        [ "depot" "checks" ];
+    in lib.setAttrByPath (baseAttrPath ++ [ name ]) {
+      description = if simulacrum then
+        "Simulacrum Test: ${name}"
+      else
+        "NixOS Test: ${name}";
       actions = {
         build = {
           description = "Build this check.";
@@ -12,10 +21,12 @@
         };
         runInteractive = {
           description = "Run interactive driver.";
-            command = lib.getExe check.driverInteractive;
+          command = if simulacrum then
+            "${pkgs.bubblewrap}/bin/bwrap --unshare-all --bind / / --dev-bind /dev /dev ${lib.getExe check.driverInteractive}"
+          else
+            lib.getExe check.driverInteractive;
         };
       };
-      }) config.checks;
-    };
+    }) config.checks);
   };
 }

packages/catalog/packages.nix

@@ -8,7 +8,7 @@ in
   perSystem = { config, ... }: {
     catalog.depot = {
       packages = lib.mapAttrs (name: package: {
-        description = package.meta.description or "Package: ${name}";
+        description = "Package: ${name}";
         actions = lib.mkMerge [
           {
             build = {

packages/checks/default.nix

@@ -7,6 +7,7 @@ let
 in
 
 {
+  debug = lib.warn "debug mode is enabled" true;
   perSystem = { filters, pkgs, self', system, ... }: {
     checks = lib.mkIf (system == "x86_64-linux") {
       ascensions = pkgs.callPackage ./ascensions.nix {

packages/checks/modules/nixos/external-storage.nix

@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+
+{
+  systemd.tmpfiles.settings."00-testing-external-storage-underlays" = lib.mapAttrs' (name: cfg: {
+    name = cfg.mountpoint;
+    value.d = {
+      user = toString cfg.uid;
+      group = toString cfg.gid;
+      mode = "0700";
+    };
+  }) config.services.external-storage.underlays;
+}

packages/checks/snakeoil/ssh/snakeoil-key

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgwAAAJAS78fWEu/H
+1gAAAAtzc2gtZWQyNTUxOQAAACAOx03X+LtW0aN8ejdN4IJgDPrTZgVwe7WbXhhBvqVwgw
+AAAEAUtGOZZIZdzGP6g85JuXBjDtciNQ9bLHNxSN5Gbwvb2Q7HTdf4u1bRo3x6N03ggmAM
++tNmBXB7tZteGEG+pXCDAAAACW1heEBUSVRBTgECAwQ=
+-----END OPENSSH PRIVATE KEY-----

packages/checks/snakeoil/ssh/snakeoil-key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA7HTdf4u1bRo3x6N03ggmAM+tNmBXB7tZteGEG+pXCD

secrets.nix

@@ -9,7 +9,6 @@ in with hosts;
   "cluster/services/dns/acme-dns-db-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ];
   "cluster/services/monitoring/secrets/grafana-db-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
   "cluster/services/monitoring/secrets/grafana-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ];
-  "cluster/services/monitoring/secrets/loki-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/monitoring/secrets/secret-monitoring/blackbox.age".publicKeys = max ++ map systemKeys [ checkmate grail prophet ];
   "cluster/services/monitoring/secrets/tempo-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ];
   "cluster/services/storage/secrets/heresy-encryption-key.age".publicKeys = max ++ map systemKeys [ VEGAS ];