hosts: restructure and modularize

This commit is contained in:
Max Headroom 2023-02-24 15:16:15 +01:00
parent be919cb2b3
commit 004af67a0e
31 changed files with 356 additions and 168 deletions

View file

@ -7,86 +7,14 @@
};
outputs = { self, nixpkgs, flake-parts, ... }@inputs:
let
flake-parts.lib.mkFlake { inherit inputs; } {
systems = [ "x86_64-linux" "aarch64-linux" ];
nixpkgsFor = nixpkgs.legacyPackages;
inherit (nixpkgs) lib;
hosts = import ./hosts;
nixosHosts' = lib.filterAttrs (_: host: host ? nixos) hosts;
nixosHosts = lib.attrNames nixosHosts';
deployableNixosHosts' = lib.filterAttrs (_: host: host ? container -> !host.container) nixosHosts';
deployableNixosHosts = lib.attrNames deployableNixosHosts';
meta = import ./tools/meta.nix;
specialArgs = {
inherit inputs hosts;
depot = inputs.self;
toolsets = import ./tools;
};
mkNixOS' = lib: name: let host = hosts.${name}; in lib.nixosSystem {
inherit specialArgs;
system = "${host.arch}-linux";
modules = [ host.nixos ./tools/inject.nix (import ./cluster/inject.nix name) ];
};
mkNixOS = mkNixOS' lib;
mkDeployEffect = branch: name: host: let
subdomain = host.enterprise.subdomain or "services";
hostname = "${lib.toLower name}.${subdomain}.${meta.domain}";
in effects.runIf (branch == "master" || branch == "staging") (effects.runNixOS {
requiredSystemFeatures = [ "hci-deploy-agent-nixos" ];
inherit (self.nixosConfigurations.${name}) config;
secretsMap.ssh = "deploy-ssh";
userSetupScript = ''
writeSSHKey ssh
cat >>~/.ssh/known_hosts <<EOF
${hostname} ${host.ssh.id.publicKey}
EOF
'';
ssh.destination = "root@${hostname}";
});
mkDeployEffects = branch: hostnames: lib.genAttrs hostnames
(name: mkDeployEffect branch name hosts.${name});
mkDeploy = name: let
host = hosts.${name};
subdomain = host.enterprise.subdomain or "services";
deploy-rs = inputs.deploy-rs.lib."${host.arch}-linux";
in {
hostname = "${lib.toLower name}.${subdomain}.${meta.domain}";
profiles.system = {
user = "root";
sshUser = "deploy";
path = deploy-rs.activate.nixos self.nixosConfigurations.${name};
};
};
mkDeployments = hosts: overrides: lib.genAttrs hosts
(host: mkDeploy host // (overrides.${host} or {}) );
effects = inputs.hercules-ci-effects.lib.withPkgs nixpkgsFor.x86_64-linux;
in flake-parts.lib.mkFlake { inherit inputs; } {
inherit systems;
flake = {
nixosConfigurations = lib.genAttrs nixosHosts mkNixOS;
deploy.nodes = mkDeployments deployableNixosHosts {};
effects = { branch, ... }: mkDeployEffects branch deployableNixosHosts;
};
imports = [
inputs.hercules-ci-effects.flakeModule
inputs.drv-parts.flakeModule
inputs.dream2nix.flakeModuleBeta
./hosts/part.nix
./modules/part.nix
./packages/part.nix
];

View file

@ -1,4 +1,5 @@
tools: {
ssh.enable = true;
ssh.id = with tools.dns; {
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICz2nGA+Y4OxhMKsV6vKIns3hOoBkK557712h7FfWXcE";
hostNames = subResolve "vegas" "backbone";
@ -15,7 +16,8 @@ tools: {
};
};
hypr = {
hyprspace = {
enable = true;
id = "QmYs4xNBby2fTs8RnzfXEk161KD4mftBfCiR8yXtgGPj4J";
addr = "10.100.3.5";
listenPort = 995;
@ -29,6 +31,6 @@ tools: {
subdomain = "backbone";
};
arch = "x86_64";
nixos = import ./system.nix;
system = "x86_64-linux";
nixos = ./system.nix;
}

View file

@ -1,6 +1,6 @@
{ config, hosts, ... }:
{ config, depot, ... }:
let
inherit (hosts.${config.networking.hostName}) interfaces;
inherit (depot.reflection) interfaces;
in
{
imports = [

View file

@ -1,10 +1,10 @@
{ hosts, ... }:
{ depot, ... }:
{
networking.nat.forwardPorts = [
{
sourcePort = 52222;
destination = "${hosts.soda.interfaces.primary.addr}:22";
destination = "${depot.config.hours.soda.interfaces.primary.addr}:22";
proto = "tcp";
}
];

View file

@ -1,4 +1,4 @@
{ config, hosts, tools, ... }:
{ config, depot, tools, ... }:
let
inherit (tools.meta) domain;
certDir = config.security.acme.certs."mail.${domain}".directory;
@ -10,7 +10,7 @@ let
lmtpSocket = "lmtp:unix:/run/dovecot2/lmtp";
postfixLdapMailboxes = "ldap:${config.age.secrets."postfix-ldap-mailboxes.cf".path}";
inherit (hosts.${config.networking.hostName}) interfaces;
inherit (depot.reflection) interfaces;
in
{
age.secrets."postfix-ldap-mailboxes.cf" = {

View file

@ -1,7 +1,7 @@
{ inputs, lib, pkgs, tools, ... }:
{ depot, lib, pkgs, tools, ... }:
let
inherit (tools.nginx) domain vhosts;
inherit (inputs.self.packages.${pkgs.system}) cinny;
inherit (depot.packages) cinny;
in
{
services.nginx.virtualHosts."chat.${domain}" = lib.recursiveUpdate

View file

@ -1,9 +1,6 @@
{ config, lib, hosts, tools, ... }:
{ config, lib, depot, tools, ... }:
let
host = hosts.${config.networking.hostName};
inherit (host) interfaces;
isNAT = interfaces.primary ? addrPublic;
inherit (depot.reflection) interfaces;
in
{
services.jitsi-meet = {
@ -30,7 +27,7 @@ in
{ type = "colibri"; }
];
};
nat = lib.optionalAttrs isNAT {
nat = lib.optionalAttrs interfaces.primary.isNat {
localAddress = interfaces.primary.addr;
publicAddress = interfaces.primary.addrPublic;
};

View file

@ -1,4 +1,4 @@
{ cluster, config, inputs, lib, pkgs, tools, ... }:
{ cluster, config, depot, lib, tools, ... }:
let
inherit (tools.meta) domain;
@ -29,7 +29,7 @@ in
};
services.grafana = {
enable = true;
package = inputs.self.packages.${pkgs.system}.grafana;
package = depot.packages.grafana;
dataDir = "/srv/storage/private/grafana";
settings = {
server = {

View file

@ -1,4 +1,4 @@
{ config, inputs, pkgs, tools, ... }:
{ config, depot, tools, ... }:
let
mkNarServe = NAR_CACHE_URL: PORT: {
@ -6,7 +6,7 @@
wantedBy = [ "multi-user.target" ];
serviceConfig = {
DynamicUser = true;
ExecStart = "${inputs.nar-serve.defaultPackage."${pkgs.system}"}/bin/nar-serve";
ExecStart = "${depot.inputs.nar-serve.packages.nar-serve}/bin/nar-serve";
};
environment = { inherit NAR_CACHE_URL PORT; };
};

View file

@ -1,4 +1,4 @@
{ config, inputs, pkgs, tools, ... }:
{ config, depot, tools, ... }:
{
links.nixIpfs.protocol = "http";
@ -6,7 +6,7 @@
systemd.services.nix-ipfs-cache = {
wantedBy = [ "multi-user.target" ];
serviceConfig = {
ExecStart = "${inputs.self.packages.${pkgs.system}.reflex-cache}/bin/reflex";
ExecStart = "${depot.packages.reflex-cache}/bin/reflex";
DynamicUser = true;
SupplementaryGroups = [ "ipfs" ];
CacheDirectory = "nix-ipfs-cache";

View file

@ -1,4 +1,4 @@
{ config, inputs, lib, pkgs, tools, ... }:
{ config, depot, lib, tools, ... }:
let
inherit (config) links;
in
@ -9,7 +9,7 @@ in
services.searx = {
enable = true;
runInUwsgi = true;
package = inputs.self.packages.${pkgs.system}.searxng;
package = depot.packages.searxng;
environmentFile = config.age.secrets.searxng-secrets.path;
settings = {
server = {

View file

@ -1,4 +1,4 @@
{ cluster, config, inputs, lib, pkgs, tools, ... }:
{ cluster, config, depot, lib, tools, ... }:
with tools.nginx;
let
login = "login.${tools.meta.domain}";
@ -32,7 +32,7 @@ in
};
services.keycloak = {
enable = true;
package = inputs.self.packages.${pkgs.system}.keycloak;
package = depot.packages.keycloak;
database = {
createLocally = false;
type = "postgresql";
@ -52,7 +52,7 @@ in
};
systemd.services.keycloak.environment = {
JAVA_OPTS = builtins.concatStringsSep " " [
"-javaagent:${inputs.self.packages.${pkgs.system}.opentelemetry-java-agent-bin}"
"-javaagent:${depot.packages.opentelemetry-java-agent-bin}"
"-Dotel.resource.attributes=service.name=keycloak"
"-Dotel.traces.exporter=otlp"
];

View file

@ -1,10 +1,8 @@
{ config, inputs, lib, pkgs, tools, ... }:
{ config, depot, lib, tools, ... }:
let
inherit (tools.meta) domain;
flakePkgs = inputs.self.packages.${pkgs.system};
link = config.links.uptime-kuma;
dataDir = "/srv/storage/private/uptime-kuma";
@ -55,7 +53,7 @@ in
ProtectKernelModules = true;
ProtectKernelTunables = true;
ExecStart = flakePkgs.uptime-kuma + /bin/uptime-kuma;
ExecStart = depot.packages.uptime-kuma + /bin/uptime-kuma;
};

View file

@ -1,9 +1,9 @@
{ inputs, pkgs, tools, ... }:
{ depot, tools, ... }:
let
importWebsites = expr: import expr {
tools = tools.nginx;
packages = inputs.self.packages.${pkgs.system};
inherit (depot) packages;
};
websites = tools.nginx.mappers.mapSubdomains (importWebsites ./websites.nix);

View file

@ -1,4 +1,4 @@
{ config, depot, hosts, inputs, tools, ... }:
{ config, depot, tools, ... }:
{
imports =
@ -11,8 +11,8 @@
./modules/oauth2-proxy
./modules/redis
./modules/virtualisation
inputs.agenix.nixosModules.age
inputs.mms.module
depot.inputs.agenix.nixosModules.age
depot.inputs.mms.module
# Services
./services/api
@ -91,7 +91,7 @@
"fe80::/10"
];
mkRules = ipt: ranges: map (x: "${ipt} -I nixos-fw 1 -d ${x} -o ${hosts.${config.networking.hostName}.interfaces.primary.link} -j DROP") ranges;
mkRules = ipt: ranges: map (x: "${ipt} -I nixos-fw 1 -d ${x} -o ${depot.reflection.interfaces.primary.link} -j DROP") ranges;
rules4 = mkRules "iptables" privateIp4Ranges;
@ -103,10 +103,10 @@
services.openssh.passwordAuthentication = false;
containers.soda = {
path = inputs.self.nixosConfigurations.soda.config.system.build.toplevel;
path = depot.nixosConfigurations.soda.config.system.build.toplevel;
privateNetwork = true;
hostBridge = "vmdefault";
localAddress = "${hosts.soda.interfaces.primary.addr}/24";
localAddress = "${depot.config.hours.soda.interfaces.primary.addr}/24";
autoStart = true;
bindMounts.sodaDir = {
hostPath = "/srv/storage/www/soda";

View file

@ -1,4 +1,5 @@
tools: {
ssh.enable = true;
ssh.id = with tools.dns; {
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINImnMfEzUBU5qiuu05DMPrddTGypOtr+cL1/yQN2GFn";
hostNames = subResolve "checkmate" "node";
@ -12,7 +13,8 @@ tools: {
};
};
hypr = {
hyprspace = {
enable = true;
id = "12D3KooWL84sAtq1QTYwb7gVbhSNX5ZUfVt4kgYKz8pdif1zpGUh";
addr = "10.100.3.32";
listenPort = 995;
@ -22,6 +24,6 @@ tools: {
subdomain = "node";
};
arch = "x86_64";
nixos = import ./system.nix;
system = "x86_64-linux";
nixos = ./system.nix;
}

View file

@ -1,4 +1,4 @@
{ aspect, inputs, hosts, ... }:
{ depot, ... }:
{
imports =
@ -6,19 +6,19 @@
# Hardware
./hardware-configuration.nix
inputs.agenix.nixosModules.age
depot.inputs.agenix.nixosModules.age
aspect.modules.hyprspace
aspect.modules.sss
]
++ aspect.sets.server;
depot.nixosModules.hyprspace
depot.nixosModules.sss
depot.nixosModules.serverBase
];
# Use the systemd-boot EFI boot loader.
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "checkmate";
networking.nameservers = [ hosts.VEGAS.interfaces.vstub.addr ];
networking.nameservers = [ depot.config.hours.VEGAS.interfaces.vstub.addr ];
time.timeZone = "Europe/Zurich";

View file

@ -1,31 +0,0 @@
let
tools = import ./tools.nix;
in with tools.dns; {
# NixOS machines
VEGAS = import ./VEGAS tools;
prophet = import ./prophet tools;
soda = import ./soda tools;
checkmate = import ./checkmate tools;
# Non-NixOS machine metadata
AnimusAlpha = let hostNames = [ "alpha.animus.com" "animus.com" ]; in {
ssh.id = {
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGpFR47Ev+W+bdng6IrDVpl8rlKBBHSr1v5lwJmZcEFH";
hostNames = portMap 69 hostNames;
};
ssh.extraConfig = tools.ssh.extraConfig hostNames [ "Port 69" ];
};
# peering
# max
TITAN.hypr = {
id = "QmfJ5Tv2z9jFv9Aocevyn6QqRcfm9eYQZhvYvmAVfACfuM";
addr = "10.100.3.7";
};
jericho.hypr = {
id = "QmccBLgGP3HR36tTkwSYZX3KDv2EXb1MvYwGVs6PbpbHv9";
addr = "10.100.3.13";
};
}

57
hosts/deploy.nix Normal file
View file

@ -0,0 +1,57 @@
{ config, inputs, lib, self, withSystem, ... }:
let
inherit (lib) elem mapAttrs toLower;
inherit (config) gods cluster defaultEffectSystem;
inherit (config.herculesCI) branch;
inherit (self) nixosConfigurations;
chosenHours = gods.fromLight;
withEffectSystem = withSystem defaultEffectSystem;
callUpon = name: host: withEffectSystem ({ hci-effects, ... }: let
inherit (hci-effects) runIf runNixOS;
inherit (host.enterprise) subdomain;
hostname = "${toLower name}.${subdomain}.${cluster.domain}";
deploy-rs = inputs.deploy-rs.lib."${host.system}";
in {
effect = runIf (elem branch [ "master" "staging" ]) (runNixOS {
requiredSystemFeatures = [ "hci-deploy-agent-nixos" ];
inherit (nixosConfigurations.${name}) config;
secretsMap.ssh = "deploy-ssh";
userSetupScript = ''
writeSSHKey ssh
cat >>~/.ssh/known_hosts <<EOF
${hostname} ${host.ssh.id.publicKey}
EOF
'';
ssh.destination = "root@${hostname}";
});
deploy = {
inherit hostname;
profiles.system = {
user = "root";
sshUser = "deploy";
path = deploy-rs.activate.nixos self.nixosConfigurations.${name};
};
};
});
calledUponHours = mapAttrs callUpon chosenHours;
pick = format: _: calledUponHour: calledUponHour.${format};
in
{
flake.effects = mapAttrs (pick "effect") calledUponHours;
flake.deploy.nodes = mapAttrs (pick "deploy") calledUponHours;
}

24
hosts/nixos.nix Normal file
View file

@ -0,0 +1,24 @@
{ config, inputs, lib, self, withSystem, ... }:
let
inherit (lib) const mapAttrs nixosSystem;
inherit (config) gods;
mkSpecialArgs = system: hostName: withSystem system ({ inputs', self', ... }: {
depot = self // self' // {
inputs = mapAttrs (name: const (inputs.${name} // inputs'.${name})) inputs;
inherit config;
# peer into the Watchman's Glass
reflection = config.hours.${hostName};
};
toolsets = import ../tools;
});
mkNixOS = name: host: nixosSystem {
specialArgs = mkSpecialArgs host.system name;
inherit (host) system;
modules = [ host.nixos ../tools/inject.nix (import ../cluster/inject.nix name) ];
};
in {
flake.nixosConfigurations = mapAttrs mkNixOS (gods.fromLight // gods.fromFlesh);
}

34
hosts/options/default.nix Normal file
View file

@ -0,0 +1,34 @@
{ config, lib, ... }:
with lib;
let
hourType = types.submodule {
imports = [
./hour/enterprise.nix
./hour/hyprspace.nix
./hour/interfaces.nix
./hour/nixos.nix
./hour/ssh.nix
];
};
mkHours = description: mkOption {
inherit description;
type = with types; attrsOf hourType;
default = {};
};
in
{
options = {
gods = {
fromLight = mkHours "Gods-from-Light: The emanations of The Glory";
fromFlesh = mkHours "Gods-from-Flesh: Mortals who penetrated the Mansus";
fromNowhere = mkHours "Gods-from-Nowhere: Lesser Hours";
};
hours = mkHours "Hours are the incarnate principles of the world." // {
readOnly = true;
default = with config.gods; fromLight // fromFlesh // fromNowhere;
};
};
}

View file

@ -0,0 +1,12 @@
{ lib, ... }:
with lib;
{
options.enterprise = {
subdomain = mkOption {
description = "Host FQDN subdomain.";
type = types.str;
default = "services";
};
};
}

View file

@ -0,0 +1,30 @@
{ lib, ... }:
with lib;
{
options.hyprspace = {
enable = mkEnableOption "Cross-host Hyprspace configuration";
id = mkOption {
description = "Hyprspace PeerID.";
type = types.str;
};
addr = mkOption {
description = "Hyprspace internal IP address.";
type = types.str;
};
routes = mkOption {
description = "Networks to export to Hyprspace.";
type = with types; listOf str;
default = [];
};
listenPort = mkOption {
description = "The port the Hyprspace daemon should listen on.";
type = types.port;
default = 8001;
};
};
}

View file

@ -0,0 +1,38 @@
{ lib, ... }:
with lib;
let
interfaceType = types.submodule ({ config, name, ... }: {
options = {
addr = mkOption {
description = "Static IP address assigned to this interface.";
type = types.str;
};
addrPublic = mkOption {
description = "Static public IP address.";
type = types.str;
default = config.addr;
};
link = mkOption {
description = "Interface link name.";
type = types.str;
default = name;
};
isNat = mkOption {
description = "Whether the host is behind NAT.";
type = types.bool;
default = config.addr != config.addrPublic;
};
};
});
in
{
options.interfaces = mkOption {
description = "Network interface information.";
type = with types; attrsOf interfaceType;
};
}

View file

@ -0,0 +1,18 @@
{ lib, ... }:
with lib;
{
options = {
system = mkOption {
description = "Nix system double for this NixOS host.";
type = types.enum systems.doubles.linux;
default = "x86_64-linux";
};
nixos = mkOption {
description = "NixOS configuration.";
type = with types; nullOr anything;
default = null;
};
};
}

View file

@ -0,0 +1,28 @@
{ lib, ... }:
with lib;
{
options.ssh = {
enable = mkEnableOption "Cross-host SSH configuration";
id = {
publicKey = mkOption {
description = "Host SSH public key.";
type = with types; nullOr str;
default = null;
};
hostNames = mkOption {
description = "Hostnames through which this host can be reached over SSH.";
type = with types; listOf str;
default = [];
};
};
extraConfig = mkOption {
description = "Extra SSH client configuration used to connect to this host.";
type = types.lines;
default = "";
};
};
}

49
hosts/part.nix Normal file
View file

@ -0,0 +1,49 @@
let
tools = import ./tools.nix;
in with tools.dns;
{
imports = [
./deploy.nix
./nixos.nix
./options
];
gods = {
fromLight = {
checkmate = import ./checkmate tools;
VEGAS = import ./VEGAS tools;
prophet = import ./prophet tools;
};
fromFlesh = {
soda = import ./soda tools;
};
fromNowhere = {
AnimusAlpha = let hostNames = [ "alpha.animus.com" "animus.com" ]; in {
ssh.enable = true;
ssh.id = {
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGpFR47Ev+W+bdng6IrDVpl8rlKBBHSr1v5lwJmZcEFH";
hostNames = portMap 69 hostNames;
};
ssh.extraConfig = tools.ssh.extraConfig hostNames [ "Port 69" ];
};
# peering
# max
TITAN.hyprspace = {
enable = true;
id = "QmfJ5Tv2z9jFv9Aocevyn6QqRcfm9eYQZhvYvmAVfACfuM";
addr = "10.100.3.7";
};
jericho.hyprspace = {
enable = true;
id = "QmccBLgGP3HR36tTkwSYZX3KDv2EXb1MvYwGVs6PbpbHv9";
addr = "10.100.3.13";
};
};
};
}

View file

@ -1,4 +1,5 @@
tools: {
ssh.enable = true;
ssh.id = with tools.dns; {
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJZ4FyGi69MksEn+UJZ87vw1APqiZmPNlEYIr0CbEoGv";
hostNames = subResolve "prophet" "node";
@ -12,7 +13,8 @@ tools: {
};
};
hypr = {
hyprspace = {
enable = true;
id = "QmbrAHuh4RYcyN9fWePCZMVmQjbaNXtyvrDCWz4VrchbXh";
addr = "10.100.3.9";
listenPort = 995;
@ -22,6 +24,6 @@ tools: {
subdomain = "node";
};
arch = "aarch64";
nixos = import ./system.nix;
system = "aarch64-linux";
nixos = ./system.nix;
}

View file

@ -1,4 +1,4 @@
{ inputs, depot, hosts, ... }:
{ depot, ... }:
{
imports =
@ -6,7 +6,7 @@
# Hardware
./hardware-configuration.nix
inputs.agenix.nixosModules.age
depot.inputs.agenix.nixosModules.age
depot.nixosModules.hyprspace
depot.nixosModules.nix-builder
@ -20,7 +20,7 @@
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "prophet";
networking.nameservers = [ hosts.VEGAS.interfaces.vstub.addr ];
networking.nameservers = [ depot.config.hours.VEGAS.interfaces.vstub.addr ];
time.timeZone = "Europe/Zurich";

View file

@ -1,4 +1,5 @@
tools: {
ssh.enable = true;
ssh.id = with tools.dns; {
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDShq3dbZy9SARsH8aSjfMQ+/eTW44eZuHVCLvgtDNKw";
hostNames = subResolve "soda" "int";
@ -16,7 +17,6 @@ tools: {
subdomain = "int";
};
arch = "x86_64";
nixos = import ./system.nix;
container = true;
system = "x86_64-linux";
nixos = ./system.nix;
}

View file

@ -1,4 +1,4 @@
{ depot, hosts, ... }:
{ depot, ... }:
{
imports = with depot.nixosModules; [
@ -14,9 +14,9 @@
networking.interfaces.eth0.useDHCP = true;
networking.nameservers = [ hosts.VEGAS.interfaces.vstub.addr ];
networking.nameservers = [ depot.config.hours.VEGAS.interfaces.vstub.addr ];
networking.resolvconf.extraConfig = "local_nameservers='${hosts.VEGAS.interfaces.vstub.addr}'";
networking.resolvconf.extraConfig = "local_nameservers='${depot.config.hours.VEGAS.interfaces.vstub.addr}'";
networking.hostName = "soda";