cluster/default.nix

@@ -17,7 +17,6 @@ lib.evalModules {
     ./lib/mesh.nix
     ./lib/secrets.nix
     ./lib/testing.nix
-    ./lib/lib.nix
 
     ./import-services.nix
   ];

cluster/lib/lib.nix

@@ -1,12 +0,0 @@
-{ config, lib, ... }:
-
-{
-  options.lib = {
-    forService = lib.mkOption {
-      description = "Enable these definitions for a particular service only.";
-      type = lib.types.functionTo lib.types.raw;
-      readOnly = true;
-      default = service: lib.mkIf (!config.simulacrum || lib.any (s: s == service) config.testConfig.activeServices);
-    };
-  };
-}

cluster/lib/testing.nix

@@ -1,15 +1,9 @@
 { lib, ... }:
 
 {
-  options = {
+  options.simulacrum = lib.mkOption {
-    simulacrum = lib.mkOption {
     description = "Whether we are in the Simulacrum.";
     type = lib.types.bool;
     default = false;
   };
-    testConfig = lib.mkOption {
-      type = lib.types.attrs;
-      readOnly = true;
-    };
-  };
 }

cluster/part.nix

@@ -3,7 +3,6 @@
 {
   imports = [
     ./catalog
-    ./simulacrum/checks.nix
   ];
 
   options.cluster = lib.mkOption {

cluster/services/wireguard/default.nix

@@ -10,19 +10,6 @@ let
   };
 
   getExtAddr = host: host.interfaces.primary.addrPublic;
-
-  snakeoilPublicKeys = {
-    checkmate = "TESTtbFybW5YREwtd18a1A4StS4YAIUS5/M1Lv0jHjA=";
-    grail = "TEsTh7bthkaDh9A1CpqDi/F121ao5lRZqIJznLH8mB4=";
-    thunderskin = "tEST6afFmVN18o+EiWNFx+ax3MJwdQIeNfJSGEpffXw=";
-    VEGAS = "tEsT6s7VtM5C20eJBaq6UlQydAha8ATlmrTRe9T5jnM=";
-    prophet = "TEstYyb5IoqSL53HbSQwMhTaR16sxcWcMmXIBPd+1gE=";
-  };
-
-  grease = hourName: realPublicKey: if config.simulacrum then
-    snakeoilPublicKeys.${hourName}
-  else
-    realPublicKey;
 in
 {
   vars = {
@@ -35,7 +22,7 @@ in
       extra = {
         meshIp = "10.1.1.32";
         inherit meshNet;
-        pubKey = grease "checkmate" "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
+        pubKey = "fZMB9CDCWyBxPnsugo3Uxm/TIDP3VX54uFoaoC0bP3U=";
         extraRoutes = [];
       };
     };
@@ -44,7 +31,7 @@ in
       extra = {
         meshIp = "10.1.1.6";
         inherit meshNet;
-        pubKey = grease "grail" "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
+        pubKey = "0WAiQGdWySsGWFUk+a9e0I+BDTKwTyWQdFT2d7BMfDQ=";
         extraRoutes = [];
       };
     };
@@ -53,7 +40,7 @@ in
       extra = {
         meshIp = "10.1.1.4";
         inherit meshNet;
-        pubKey = grease "thunderskin" "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
+        pubKey = "xvSsFvCVK8h2wThZJ7E5K0fniTBIEIYOblkKIf3Cwy0=";
         extraRoutes = [];
       };
     };
@@ -62,7 +49,7 @@ in
       extra = {
         meshIp = "10.1.1.5";
         inherit meshNet;
-        pubKey = grease "VEGAS" "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
+        pubKey = "NpeB8O4erGTas1pz6Pt7qtY9k45YV6tcZmvvA4qXoFk=";
         extraRoutes = [ "${hours.VEGAS.interfaces.vstub.addr}/32" "10.10.0.0/16" ];
       };
     };
@@ -71,7 +58,7 @@ in
       extra = {
         meshIp = "10.1.1.9";
         inherit meshNet;
-        pubKey = grease "prophet" "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
+        pubKey = "MMZAbRtNE+gsLm6DJy9VN/Y39E69oAZnvOcFZPUAVDc=";
         extraRoutes = [];
       };
     };
@@ -82,12 +69,8 @@ in
       storm = [ "VEGAS" ];
     };
     nixos = {
-      mesh = [
+      mesh = ./mesh.nix;
-        ./mesh.nix
+      storm = ./storm.nix;
-      ] ++ lib.optionals config.simulacrum [
-        ./simulacrum/snakeoil-keys.nix
-      ];
-      storm = [ ./storm.nix ];
     };
     secrets.meshPrivateKey = {
       nodes = config.services.wireguard.nodes.mesh;

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-VEGAS

@@ -1 +0,0 @@
-MNvWpMluuzQvPyGTp7jtyPSyz6n9lIly/WX1gW2NAHg=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-checkmate

@@ -1 +0,0 @@
-YHzP8rBP6qiXs6ZdnvHop9KnCYRADIEejwZzAzvj8m4=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-grail

@@ -1 +0,0 @@
-uD7X5E6N9d0sN+xPr/bWnehSa3bAok741GO7Z4I+Z3I=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-prophet

@@ -1 +0,0 @@
-QHyIJ3HoKGGFN28qOrQP4UyoQMP5bM7Idn2MzayKzEM=

cluster/services/wireguard/simulacrum/keys/snakeoilPrivateKey-thunderskin

@@ -1 +0,0 @@
-YLl+hkWaCWx/5PpWs3cQ+bKqYdJef/qZ+FMTsM9ammM=

cluster/services/wireguard/simulacrum/snakeoil-keys.nix

@@ -1,6 +0,0 @@
-{ lib, config, ... }: {
-  config.environment.etc = {
-    "dummy-secrets/cluster-wireguard-meshPrivateKey".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
-    "dummy-secrets/wireguard-key-storm".source = lib.mkForce ./keys/snakeoilPrivateKey-${config.networking.hostName};
-  };
-}

cluster/simulacrum/checks.nix

@@ -1,16 +0,0 @@
-{ config, extendModules, lib, ... }:
-
-{
-  perSystem = { pkgs, system, ... }: {
-    checks = lib.mkIf (system == "x86_64-linux") (lib.mapAttrs' (name: svc: let
-      runSimulacrum = pkgs.callPackage ./. {
-        inherit config extendModules;
-      };
-    in {
-      name = "simulacrum-${name}";
-      value = runSimulacrum {
-        service = name;
-      };
-    }) (lib.filterAttrs (_: svc: svc.simulacrum.enable) config.cluster.config.services));
-  };
-}

cluster/simulacrum/default.nix

@@ -1,134 +0,0 @@
-{ testers, config, extendModules, lib, system }:
-
-{ service }:
-
-let
-  serviceConfig = config.cluster.config.services.${service};
-  serviceList = getDepsRecursive [] service;
-  allAugments = map (svc: config.cluster.config.services.${svc}.simulacrum.augments) serviceList;
-
-  getDepsRecursive = acc: service: let
-    deps = lib.subtractLists acc config.cluster.config.services.${service}.simulacrum.deps;
-    acc' = acc ++ [ service ];
-    recurse = getDepsRecursive acc';
-  in lib.unique (lib.flatten ([ service ] ++ map recurse deps));
-
-  lift = config;
-
-  snakeoil = {
-    ssh = {
-      public = lib.fileContents ../../packages/checks/snakeoil/ssh/snakeoil-key.pub;
-      private = ../../packages/checks/snakeoil/ssh/snakeoil-key;
-    };
-  };
-
-  nodes = lib.attrNames config.gods.fromLight;
-  nodes' = lib.attrNames (config.gods.fromLight // { nowhere = null; });
-  digits = lib.attrsets.listToAttrs (lib.zipListsWith lib.nameValuePair nodes' (lib.range 1 255));
-  depot' = extendModules {
-    modules = [
-      ({ config, ... }: {
-        gods.fromLight = lib.mapAttrs (name: cfg: {
-          interfaces.primary = {
-            link = lib.mkForce "vprimary";
-          };
-          ssh.id.publicKey = lib.mkForce snakeoil.ssh.public;
-        }) lift.gods.fromLight;
-
-        cluster = lib.mkForce (lift.cluster.extendModules {
-          specialArgs.depot = config;
-          modules = [
-            {
-              simulacrum = true;
-              testConfig = {
-                subject = service;
-                activeServices = serviceList;
-              };
-            }
-          ];
-        });
-      })
-    ];
-  };
-  specialArgs = depot'.config.lib.summon system lib.id;
-in
-
-testers.runNixOSTest {
-  name = "simulacrum-${service}";
-
-  imports = [
-    serviceConfig.simulacrum.settings
-    ./nowhere
-    {
-      nodes.nowhere.imports = [
-        config.flake.nixosModules.port-magic
-      ];
-    }
-  ] ++ allAugments;
-
-  _module.args = {
-    inherit (depot'.config) cluster;
-  };
-
-  node = { inherit specialArgs; };
-  nodes = lib.genAttrs nodes (node: let
-    hour = depot'.config.hours.${node};
-  in {
-    imports = [
-      specialArgs.depot.hours.${node}.nixos
-      ../../packages/checks/modules/nixos/age-dummy-secrets
-      ../../packages/checks/modules/nixos/external-storage.nix
-    ] ++ depot'.config.cluster.config.out.injectNixosConfigForServices serviceList node;
-
-    boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
-    networking = {
-      interfaces = {
-        ${hour.interfaces.primary.link} = {
-          useDHCP = lib.mkForce false;
-          virtual = true;
-          ipv4.addresses = lib.mkForce ([
-            {
-              address = hour.interfaces.primary.addr;
-              prefixLength = 32;
-            }
-          ] ++ lib.optional hour.interfaces.primary.isNat {
-            address = hour.interfaces.primary.addrPublic;
-            prefixLength = 32;
-          });
-        };
-        eth1.ipv4.routes = lib.pipe nodes [
-          (lib.filter (n: n != node))
-          (map (n: let
-            hour = depot'.config.hours.${n};
-          in {
-            address = hour.interfaces.primary.addrPublic;
-            prefixLength = 32;
-            via = "192.168.1.${toString digits.${n}}";
-          }))
-        ];
-      };
-
-      firewall.extraCommands = lib.mkAfter (lib.optionalString (hour.interfaces.primary.isNat) ''
-        # self-nat
-        iptables -t nat -A PREROUTING -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
-        iptables -t nat -A OUTPUT -d ${hour.interfaces.primary.addrPublic} -j DNAT --to-destination ${hour.interfaces.primary.addr}
-        iptables -t nat -A POSTROUTING -s ${hour.interfaces.primary.addr} -j SNAT --to-source ${hour.interfaces.primary.addrPublic}
-      '');
-    };
-
-    systemd.services = {
-      hyprspace.enable = false;
-    };
-
-    environment.etc = {
-      "ssh/ssh_host_ed25519_key" = {
-        source = snakeoil.ssh.private;
-        mode = "0400";
-      };
-    };
-    virtualisation = {
-      cores = 2;
-      memorySize = 4096;
-    };
-  });
-}

cluster/simulacrum/nowhere/default.nix

@@ -1,101 +0,0 @@
-{ cluster, config, lib, pkgs, ... }:
-
-let
-  lift = config;
-
-  cfsslConfigIntermediateCA = pkgs.writeText "simulacrum-cfssl-config.json" (builtins.toJSON {
-    signing = {
-      default.expiry = "8760h";
-      profiles.intermediate = {
-        expiry = "8760h";
-        usages = [
-          "cert sign"
-          "crl sign"
-        ];
-        ca_constraint = {
-          is_ca = true;
-          max_path_len = 1;
-        };
-      };
-    };
-  });
-
-  caCsr = pkgs.writeText "simulacrum-ca-csr.json" (builtins.toJSON {
-    CN = "Simulacrum Root CA";
-  });
-
-  ca = pkgs.runCommand "simulacrum-snakeoil-ca" {
-    nativeBuildInputs = [
-      pkgs.cfssl
-    ];
-  } ''
-    mkdir $out
-    cfssl gencert --initca ${caCsr} | cfssljson --bare $out/ca
-  '';
-
-  genCert = extraFlags: csrData: let
-    csr = pkgs.writeText "simulacrum-csr.json" (builtins.toJSON csrData);
-  in pkgs.runCommand "simulacrum-snakeoil-cert" {
-    nativeBuildInputs = [
-      pkgs.cfssl
-    ];
-  } ''
-    mkdir $out
-    cfssl gencert ${lib.escapeShellArgs ([
-      "--ca=file:${ca}/ca.pem"
-      "--ca-key=file:${ca}/ca-key.pem"
-    ] ++ extraFlags ++ [
-      csr
-    ])} | cfssljson --bare $out/cert
-  '';
-
-  genHostCert = hostname: genCert [ "--hostname=${hostname}" ] { CN = hostname; };
-
-  getNodeAddr = node: (builtins.head config.nodes.${node}.networking.interfaces.eth1.ipv4.addresses).address;
-in
-
-{
-  imports = [
-    ./options.nix
-  ];
-  defaults = {
-    networking.hosts."${getNodeAddr "nowhere"}" = lib.attrNames config.nowhere.names;
-    security.pki.certificateFiles = [
-      "${ca}/ca.pem"
-    ];
-  };
-
-  nowhere.certs = {
-    inherit ca;
-    intermediate = genCert [ "--config=${cfsslConfigIntermediateCA}" "--profile=intermediate" ] {
-      CN = "Simulacrum Intermediate CA";
-    };
-  };
-
-  nodes.nowhere = { config, depot, ... }: {
-    networking = {
-      firewall.allowedTCPPorts = [ 443 ];
-      interfaces.eth1.ipv4.routes = lib.mapAttrsToList (name: hour: {
-        address = hour.interfaces.primary.addrPublic;
-        prefixLength = 32;
-        via = getNodeAddr name;
-      }) depot.gods.fromLight;
-      nameservers = map (name: depot.hours.${name}.interfaces.primary.addrPublic) cluster.config.services.dns.nodes.authoritative;
-    };
-    services.nginx = {
-      enable = true;
-      recommendedProxySettings = true;
-      virtualHosts = lib.mapAttrs (name: link: let
-        cert = genHostCert name;
-      in {
-        forceSSL = true;
-        sslCertificate = "${cert}/cert.pem";
-        sslCertificateKey = "${cert}/cert-key.pem";
-        locations."/" = {
-          proxyPass = config.links.${link}.url;
-          extraConfig = "proxy_ssl_verify off;";
-        };
-      }) lift.nowhere.names;
-    };
-  };
-}

cluster/simulacrum/nowhere/options.nix

@@ -1,16 +0,0 @@
-{ lib, ... }:
-
-{
-  options.nowhere = {
-    names = lib.mkOption {
-      description = "Hostnames that point Nowhere.";
-      type = with lib.types; attrsOf str;
-      default = {};
-    };
-    certs = lib.mkOption {
-      description = "Snakeoil certificate packages.";
-      type = with lib.types; attrsOf package;
-      default = {};
-    };
-  };
-}

packages/catalog/checks.nix

@@ -1,19 +1,10 @@
 { lib, ... }:
 
 {
-  perSystem = { config, pkgs, ... }: {
+  perSystem = { config, ... }: {
-    catalog = lib.mkMerge (lib.mapAttrsToList (name': check: let
+    catalog.depot = {
-      simulacrum = lib.hasPrefix "simulacrum-" name';
+      checks = lib.mapAttrs (name: check: {
-      name = lib.removePrefix "simulacrum-" name';
+        description = "NixOS Test: ${name}";
-      baseAttrPath = if simulacrum then
-        [ "cluster" "simulacrum" ]
-      else
-        [ "depot" "checks" ];
-    in lib.setAttrByPath (baseAttrPath ++ [ name ]) {
-      description = if simulacrum then
-        "Simulacrum Test: ${name}"
-      else
-        "NixOS Test: ${name}";
         actions = {
           build = {
             description = "Build this check.";
@@ -21,12 +12,10 @@
           };
           runInteractive = {
             description = "Run interactive driver.";
-          command = if simulacrum then
+            command = lib.getExe check.driverInteractive;
-            "${pkgs.bubblewrap}/bin/bwrap --unshare-all --bind / / --dev-bind /dev /dev ${lib.getExe check.driverInteractive}"
-          else
-            lib.getExe check.driverInteractive;
           };
         };
-    }) config.checks);
+      }) config.checks;
+    };
   };
 }

packages/checks/default.nix

@@ -15,6 +15,12 @@ in
         inherit (config) cluster;
       };
 
+      garage = pkgs.callPackage ./garage.nix {
+        inherit (self'.packages) garage consul;
+        inherit (self) nixosModules;
+        inherit (config) cluster;
+      };
+
       ipfs-cluster-upgrade = pkgs.callPackage ./ipfs-cluster-upgrade.nix {
         inherit (self) nixosModules;
         previous = timeMachine.preUnstable;

packages/checks/garage.nix

@@ -0,0 +1,155 @@
+{ testers, nixosModules, cluster, garage, consul }:
+
+testers.runNixOSTest {
+  name = "garage";
+
+  imports = [
+    ./modules/consul.nix
+  ];
+
+  extraBaseModules.services.consul.package = consul;
+
+  nodes = let
+    common = { config, lib, ... }: let
+      inherit (config.networking) hostName primaryIPAddress;
+    in {
+      imports = lib.flatten [
+        ./modules/nixos/age-dummy-secrets
+        ./modules/nixos/age-dummy-secrets/options.nix
+        nixosModules.ascensions
+        nixosModules.systemd-extras
+        nixosModules.consul-distributed-services
+        nixosModules.port-magic
+        cluster.config.services.storage.nixos.garage
+        cluster.config.services.storage.nixos.garageInternal
+        cluster.config.services.consul.nixos.ready
+      ];
+      options.services.locksmith.providers = lib.mkOption {
+        type = lib.types.raw;
+      };
+      config = {
+        links.consulAgent = {
+          protocol = "http";
+          hostname = "consul";
+          port = 8500;
+        };
+        _module.args = {
+          depot.packages = { inherit garage; };
+          cluster.config = {
+            hostLinks.${hostName} = {
+              garageRpc.tuple = "${primaryIPAddress}:3901";
+              garageS3.tuple = "${primaryIPAddress}:8080";
+              garageWeb.tuple = "${primaryIPAddress}:8081";
+            };
+            links.garageWeb.hostname = "web.garage.example.com";
+            vars.meshNet.cidr = "192.168.0.0/16";
+          };
+        };
+        environment.etc."dummy-secrets/garageRpcSecret".text = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+        networking.firewall.allowedTCPPorts = [ 3901 8080 ];
+        services.garage = {
+          layout.initial = lib.mkOverride 51 {
+            garage1 = { zone = "dc1"; capacity = 1000; };
+            garage2 = { zone = "dc1"; capacity = 1000; };
+            garage3 = { zone = "dc1"; capacity = 1000; };
+          };
+        };
+        system.ascensions.garage-layout.incantations = lib.mkOverride 51 (i: [ ]);
+        specialisation.modifiedLayout = {
+          inheritParentConfig = true;
+          configuration = {
+            services.garage = {
+              layout.initial = lib.mkForce {
+                garage1 = { zone = "dc1"; capacity = 2000; };
+                garage2 = { zone = "dc1"; capacity = 1000; };
+                garage3 = { zone = "dc1"; capacity = 1000; };
+              };
+              keys.testKey.allow.createBucket = true;
+              buckets = {
+                bucket1 = {
+                  allow.testKey = [ "read" "write" ];
+                  quotas = {
+                    maxObjects = 300;
+                    maxSize = 400 * 1024 * 1024;
+                  };
+                };
+                bucket2 = {
+                  allow.testKey = [ "read" ];
+                };
+              };
+            };
+            system.ascensions.garage-layout.incantations = lib.mkForce (i: [
+              (i.runGarage ''
+                garage layout assign -z dc1 -c 2000 "$(garage node id -q | cut -d@ -f1)"
+                garage layout apply --version 2
+              '')
+            ]);
+          };
+        };
+      };
+    };
+  in {
+    garage1.imports = [ common ];
+    garage2.imports = [ common ];
+    garage3.imports = [ common ];
+  };
+
+  testScript = { nodes, ... }: /*python*/ ''
+    nodes = [garage1, garage2, garage3]
+
+    start_all()
+
+    with subtest("should bootstrap new cluster"):
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+    with subtest("should apply new layout with ascension"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl list-jobs | wc -l)" -eq 1')
+
+      for node in nodes:
+        node.succeed("/run/current-system/specialisation/modifiedLayout/bin/switch-to-configuration test")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should apply new layout from scratch"):
+      for node in nodes:
+        node.systemctl("stop garage.service")
+        node.succeed("rm -rf /var/lib/garage-metadata")
+
+      for node in nodes:
+        node.systemctl("start garage.service")
+
+      for node in nodes:
+        node.wait_for_unit("garage.service")
+
+      for node in nodes:
+        node.wait_until_fails("garage status | grep 'NO ROLE ASSIGNED'")
+
+      for node in nodes:
+        node.wait_until_succeeds("garage layout show | grep -w 2000")
+        assert "1" in node.succeed("garage layout show | grep -w 2000 | wc -l")
+        assert "2" in node.succeed("garage layout show | grep -w 1000 | wc -l")
+
+    with subtest("should create specified buckets and keys"):
+      for node in nodes:
+        node.wait_until_succeeds('test "$(systemctl is-active garage-apply)" != activating')
+      garage1.succeed("garage key list | grep testKey")
+      garage1.succeed("garage bucket list | grep bucket1")
+      garage1.succeed("garage bucket list | grep bucket2")
+
+    with subtest("should delete unspecified buckets and keys"):
+      garage1.succeed("garage bucket create unwantedbucket")
+      garage1.succeed("garage key new --name unwantedkey")
+      garage1.succeed("systemctl restart garage-apply.service")
+
+      garage1.fail("garage key list | grep unwantedkey")
+      garage1.fail("garage bucket list | grep unwantedbucket")
+  '';
+}