privatevoid.net/depot
1
1
Fork 0
Code Issues 18 Pull requests 2 Projects 3 Releases Packages Wiki Activity Actions

Ways #98

Merged
max merged 10 commits from pr-cluster-reverse-proxy into master 2024-07-04 22:05:22 +03:00
Conversation 6 Commits 10 Files changed 17 +339 -165
17 changed files with 339 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cluster/default.nix
View file

@ -14,7 +14,8 @@ lib.evalModules {
./lib/services.nix
./lib/inject-nixos-config.nix
./lib/port-magic-multi.nix
./lib/mesh.nix
./import-services.nix
];
}
}

15
cluster/lib/mesh.nix Normal file
View file

@ -0,0 +1,15 @@
{ config, lib, ... }:
{
hostLinks = lib.pipe config.services [
(lib.filterAttrs (_: svc: svc.meshLinks != {}))
(lib.mapAttrsToList (svcName: svc: lib.mapAttrsToList (name: cfg: lib.genAttrs svc.nodes.${name} (hostName: {
${cfg.name} = { ... }: {
imports = [ cfg.link ];
ipv4 = config.vars.mesh.${hostName}.meshIp;
};
})) svc.meshLinks))
(map lib.mkMerge)
lib.mkMerge
];
}

19
cluster/lib/service-module.nix
View file

@ -1,8 +1,9 @@
{ config, lib, ... }:
{ config, lib, name, ... }:
with lib;
let
filterGroup = group: hostName: builtins.filter (x: x != hostName) group;
serviceName = name;
in
{
@ -31,6 +32,22 @@ in
type = with types; attrs;
default = {};
};
meshLinks = mkOption {
description = "Create host links on the mesh network.";
type = types.attrsOf (types.submodule ({ name, ... }: {
options = {
name = mkOption {
type = types.str;
default = "${serviceName}-${name}";
};
link = mkOption {
type = types.deferredModule;
default = {};
};
};
}));
default = {};
};
};
config.otherNodes = builtins.mapAttrs (const filterGroup) config.nodes;
}

10
cluster/services/forge/default.nix
View file

@ -4,11 +4,15 @@
services.forge = {
nodes.server = [ "VEGAS" ];
nixos.server = ./server.nix;
meshLinks.server = {
name = "forge";
link.protocol = "http";
};
};
dns.records.forge.target = map
(node: depot.hours.${node}.interfaces.primary.addrPublic)
config.services.forge.nodes.server;
ways.forge.target = let
host = builtins.head config.services.forge.nodes.server;
in config.hostLinks.${host}.forge.url;
garage = {
keys.forgejo = { };

6
cluster/services/forge/server.nix
View file

@ -9,7 +9,7 @@ let
host = "forge.${domain}";
link = config.links.forge;
link = cluster.config.hostLinks.${config.networking.hostName}.forge;
exe = lib.getExe config.services.forgejo.package;
in
@ -43,8 +43,6 @@ in
};
};
links.forge.protocol = "http";
services.forgejo = {
enable = true;
package = depot.packages.forgejo;
@ -101,8 +99,6 @@ in
};
};
services.nginx.virtualHosts."${host}" = vhosts.proxy link.url;
systemd.services.forgejo.preStart = let
providerName = "PrivateVoidAccount";
args = lib.escapeShellArgs [

35
cluster/services/ipfs/default.nix
View file

@ -29,6 +29,10 @@
io-tweaks = [ "VEGAS" ];
remote-api = [ "VEGAS" ];
};
meshLinks.gateway = {
name = "ipfsGateway";
link.protocol = "http";
};
nixos = {
node = [
./node.nix
@ -51,18 +55,37 @@
};
dns.records = {
p2p.consulService = "ipfs-gateway";
pin.consulService = "ipfs-gateway";
"ipfs.admin".target = map
(node: depot.hours.${node}.interfaces.primary.addrPublic)
config.services.ipfs.nodes.remote-api;
"^[^_].+\\.ipfs" = {
pin.consulService = "ipfs-gateway";
};
ways = {
p2p = {
consulService = "ipfs-gateway";
rewrite.type = "regex";
extras.locations."/routing" = {
extraConfig = ''
add_header X-Content-Type-Options "";
add_header Access-Control-Allow-Origin *;
'';
};
};
"^[^_].+\\.ipns" = {
ipfs = {
consulService = "ipfs-gateway";
rewrite.type = "regex";
wildcard = true;
extras.extraConfig = ''
add_header X-Content-Type-Options "";
add_header Access-Control-Allow-Origin *;
'';
};
ipns = {
consulService = "ipfs-gateway";
wildcard = true;
extras.extraConfig = ''
add_header X-Content-Type-Options "";
add_header Access-Control-Allow-Origin *;
'';
};
};
}

56
cluster/services/ipfs/gateway.nix
View file

@ -1,8 +1,7 @@
{ config, depot, lib, ... }:
with depot.lib.nginx;
{ cluster, config, depot, lib, ... }:
let
inherit (depot.lib.meta) domain;
gw = config.links.ipfsGateway;
gw = cluster.config.hostLinks.${config.networking.hostName}.ipfsGateway;
cfg = config.services.ipfs;
metrics = config.links.ipfsMetrics;
in
@ -34,42 +33,6 @@ in
locations."/".return = "204";
locations."${metrics.path}".proxyPass = "http://unix:/run/ipfs/ipfs-api.sock:";
};
"p2p.${domain}" = vhosts.basic // {
locations."/".return = "204";
locations."/routing" = {
proxyPass = gw.url;
extraConfig = ''
add_header X-Content-Type-Options "";
add_header Access-Control-Allow-Origin *;
'';
};
};
};
security.acme.certs."ipfs.${domain}" = {
domain = "*.ipfs.${domain}";
extraDomainNames = [ "*.ipns.${domain}" ];
dnsProvider = "exec";
group = "nginx";
};
security.acme.certs."p2p.${domain}" = {
dnsProvider = "exec";
webroot = lib.mkForce null;
};
services.nginx.virtualHosts."ipfs.${domain}" = vhosts.basic // {
serverName = "~^(.+)\.(ip[fn]s)\.${domain}$";
enableACME = false;
useACMEHost = "ipfs.${domain}";
locations = {
"/" = {
proxyPass = gw.url;
extraConfig = ''
add_header X-Content-Type-Options "";
add_header Access-Control-Allow-Origin *;
'';
};
};
};
services.ipfs.extraConfig.Gateway.PublicGateways = {
@ -88,20 +51,11 @@ in
consul.services.ipfs-gateway = {
mode = "external";
unit = "ipfs";
definition = rec {
definition = {
name = "ipfs-gateway";
address = depot.reflection.interfaces.primary.addrPublic;
port = 443;
address = gw.ipv4;
port = gw.port;
checks = [
rec {
name = "Frontend";
id = "service:ipfs-gateway:frontend";
interval = "60s";
http = "https://${address}/";
tls_server_name = "bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354.ipfs.${domain}"; # empty directory
header.Host = lib.singleton tls_server_name;
method = "HEAD";
}
{
name = "IPFS Node";
id = "service:ipfs-gateway:ipfs";

8
cluster/services/ipfs/node.nix
View file

@ -4,7 +4,7 @@ let
cfg = config.services.ipfs;
apiAddress = "/unix/run/ipfs/ipfs-api.sock";
ipfsApi = pkgs.writeTextDir "api" apiAddress;
gw = config.links.ipfsGateway;
gw = cluster.config.hostLinks.${config.networking.hostName}.ipfsGateway;
ipfsPort = 110;
nameservers = lib.unique config.networking.nameservers;
in
@ -13,8 +13,6 @@ in
depot.nixosModules.ipfs
];
links.ipfsGateway.protocol = "http";
networking.firewall = {
allowedTCPPorts = [ ipfsPort 4001 ];
allowedUDPPorts = [ ipfsPort 4001 ];
@ -157,7 +155,9 @@ in
"fc00::/7"
"fe80::/10"
];
IPAddressAllow = nameservers;
IPAddressAllow = nameservers ++ [
cluster.config.vars.meshNet.cidr
];
};
postStart = "chmod 660 /run/ipfs/ipfs-api.sock";
};

4
cluster/services/ipfs/remote-api.nix
View file

@ -1,9 +1,9 @@
{ config, depot, ... }:
{ cluster, config, depot, ... }:
with depot.lib.nginx;
let
inherit (depot.lib.meta) domain;
cfg = config.services.ipfs;
gw = config.links.ipfsGateway;
gw = cluster.config.hostLinks.${config.networking.hostName}.ipfsGateway;
in
{
users.users.nginx.extraGroups = [ cfg.group ];

5
cluster/services/monitoring/default.nix
View file

@ -85,5 +85,8 @@ in
};
};
dns.records.monitoring.consulService = "grafana";
ways.monitoring = {
consulService = "grafana";
extras.locations."/".proxyWebsockets = true;
};
}

31
cluster/services/monitoring/grafana-ha.nix
View file

@ -8,8 +8,6 @@ let
inherit (config.networking) hostName;
svc = cluster.config.services.monitoring;
iniList = lib.concatStringsSep " ";
login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}";
@ -93,36 +91,15 @@ in
};
};
services.nginx = {
upstreams.grafana-ha.servers = lib.mapAttrs' (_: links: lib.nameValuePair links.grafana.tuple {}) (lib.getAttrs (svc.nodes.grafana) hostLinks);
virtualHosts."monitoring.${domain}" = lib.recursiveUpdate (depot.lib.nginx.vhosts.proxy "http://grafana-ha") {
locations."/".proxyWebsockets = true;
};
};
security.acme.certs."monitoring.${domain}" = {
dnsProvider = "exec";
webroot = lib.mkForce null;
};
consul.services.grafana = {
mode = "manual";
definition = rec {
definition = {
name = "grafana";
address = depot.reflection.interfaces.primary.addrPublic;
port = 443;
address = hostLinks.${hostName}.grafana.ipv4;
port = hostLinks.${hostName}.grafana.port;
checks = [
rec {
name = "Frontend";
id = "service:grafana:frontend";
interval = "30s";
http = "https://${address}/healthz";
tls_server_name = "monitoring.${domain}";
header.Host = lib.singleton tls_server_name;
}
{
name = "Backend";
name = "Grafana";
id = "service:grafana:backend";
interval = "5s";
http = "${hostLinks.${hostName}.grafana.url}/healthz";

18
cluster/services/storage/default.nix
View file

@ -88,9 +88,19 @@ in
};
};
dns.records.garage.consulService = "garage";
dns.records."^[^_].+\\.web\\.garage" = {
consulService = "garage-web";
rewrite.type = "regex";
ways = {
garage = {
consulService = "garage";
extras.extraConfig = ''
client_max_body_size 4G;
'';
};
"web.garage" = {
consulService = "garage-web";
wildcard = true;
extras.locations."/".extraConfig = ''
proxy_set_header Host "$1.${config.links.garageWeb.hostname}";
'';
};
};
}

66
cluster/services/storage/garage-gateway.nix
View file

@ -1,8 +1,8 @@
{ config, cluster, depot, lib, ... }:
let
linkS3 = cluster.config.links.garageS3;
linkWeb = cluster.config.links.garageWeb;
linkS3 = cluster.config.hostLinks.${config.networking.hostName}.garageS3;
linkWeb = cluster.config.hostLinks.${config.networking.hostName}.garageWeb;
in
{
@ -10,60 +10,14 @@ in
services.garage.settings.admin.api_bind_addr = config.links.garageMetrics.tuple;
services.nginx.virtualHosts = {
${linkS3.hostname} = depot.lib.nginx.vhosts.basic // {
locations = {
"/".proxyPass = cluster.config.hostLinks.${config.networking.hostName}.garageS3.url;
"= /health".proxyPass = config.links.garageMetrics.url;
};
extraConfig = "client_max_body_size 4G;";
};
"${linkWeb.hostname}" = depot.lib.nginx.vhosts.basic // {
serverName = "~^(.+)\.${lib.escapeRegex linkWeb.hostname}$";
enableACME = false;
useACMEHost = linkWeb.hostname;
locations = {
"/" = {
proxyPass = cluster.config.hostLinks.${config.networking.hostName}.garageWeb.url;
extraConfig = ''
proxy_set_header Host "$1.${linkWeb.hostname}";
'';
};
"= /.internal-api/garage/health" = {
proxyPass = "${config.links.garageMetrics.url}/health";
};
};
};
};
security.acme.certs = {
${linkS3.hostname} = {
dnsProvider = "exec";
webroot = lib.mkForce null;
};
${linkWeb.hostname} = {
domain = "*.${linkWeb.hostname}";
dnsProvider = "exec";
group = "nginx";
};
};
consul.services = {
garage = {
mode = "external";
definition = rec {
definition = {
name = "garage";
address = depot.reflection.interfaces.primary.addrPublic;
address = linkS3.ipv4;
inherit (linkS3) port;
checks = [
{
name = "Frontend";
id = "service:garage:frontend";
interval = "60s";
http = "https://${address}/health";
tls_server_name = linkS3.hostname;
header.Host = lib.singleton linkS3.hostname;
}
{
name = "Garage Node";
id = "service:garage:node";
@ -76,19 +30,11 @@ in
garage-web = {
mode = "external";
unit = "garage";
definition = rec {
definition = {
name = "garage-web";
address = depot.reflection.interfaces.primary.addrPublic;
address = linkWeb.ipv4;
inherit (linkWeb) port;
checks = [
{
name = "Frontend";
id = "service:garage-web:frontend";
interval = "60s";
http = "https://${address}/.internal-api/garage/health";
tls_server_name = "healthcheck.${linkWeb.hostname}";
header.Host = lib.singleton "healthcheck.${linkWeb.hostname}";
}
{
name = "Garage Node";
id = "service:garage-web:node";

16
cluster/services/ways/default.nix Normal file
View file

@ -0,0 +1,16 @@
{ config, lib, ... }:
{
imports = [
./options
];
services.ways = {
nodes.host = config.services.websites.nodes.host;
nixos.host = ./host.nix;
};
dns.records = lib.mapAttrs'
(_: cfg: lib.nameValuePair cfg.dnsRecord.name ({ ... }: { imports = [ cfg.dnsRecord.value ]; }))
config.ways;
}

103
cluster/services/ways/host.nix Normal file
View file

@ -0,0 +1,103 @@
{ cluster, config, depot, lib, pkgs, ... }:
let
externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;
consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
consulHttpAddr = "${config.services.consul.extraConfig.addresses.http or "127.0.0.1"}:${toString (config.services.consul.extraConfig.ports.http or 8500)}";
in
{
services.nginx = {
virtualHosts = lib.mapAttrs' (name: cfg: {
name = cfg.name;
value = { ... }: {
imports = [
cfg.extras
{
forceSSL = true;
enableACME = !cfg.internal && !cfg.wildcard;
useACMEHost = lib.mkMerge [
(lib.mkIf cfg.internal cfg.domainSuffixInternal)
(lib.mkIf cfg.wildcard "${name}.${cfg.domainSuffix}")
];
locations = lib.mkMerge [
{
"/".proxyPass = cfg.target;
"${cfg.healthCheckPath}".extraConfig = "access_log off;";
}
{
"/.well-known/ways/internal-health-check" = {
return = ''200 "INTERNAL_OK\n"'';
extraConfig = "access_log off;";
};
}
];
}
];
};
}) cluster.config.ways;
appendHttpConfig = lib.mkIf (consulServiceWays != {}) ''
include /run/consul-template/nginx-ways-*.conf;
'';
};
security.acme.certs = lib.mapAttrs' (name: cfg: {
name = "${name}.${cfg.domainSuffix}";
value = {
domain = lib.mkIf cfg.wildcard "*.${name}.${cfg.domainSuffix}";
dnsProvider = "exec";
webroot = lib.mkForce null;
group = "nginx";
};
}) externalWays;
systemd.services = lib.mapAttrs' (name: cfg: {
name = "acme-${name}.${cfg.domainSuffix}";
value.distributed.enable = true;
}) externalWays;
services.consul-template.instances.ways = lib.mkIf (consulServiceWays != {}) {
user = "nginx";
group = "nginx";
settings = {
consul.address = "http://${consulHttpAddr}";
template = [
{
source = let
upstreams = lib.mapAttrsToList (_: cfg: ''
upstream ${cfg.nginxUpstreamName} {
{{ range $i, $e := service "${cfg.consulService}~_agent" -}}
server {{ .Address }}:{{ .Port }}{{ if ne $i 0 }} backup{{ end }};
{{end}}
}
'') consulServiceWays;
in pkgs.writeText "ways-upstreams.ctmpl" (lib.concatStringsSep "\n" (lib.unique upstreams));
destination = "/run/consul-template/nginx-ways-upstreams.conf";
exec.command = [
"${config.services.nginx.package}/bin/nginx"
"-s" "reload"
"-g" "pid /run/nginx/nginx.pid;"
];
}
];
};
};
consul.services.ways-proxy = {
unit = "nginx";
mode = "external";
definition = {
name = "ways-proxy";
address = depot.reflection.interfaces.primary.addrPublic;
port = 443;
checks = lib.singleton {
interval = "60s";
tcp = "127.0.0.1:80";
};
tags = lib.attrNames externalWays;
};
};
}

12
cluster/services/ways/options/default.nix Normal file
View file

@ -0,0 +1,12 @@
{ lib, depot, ... }:
{
options.ways = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule {
imports = [ ./way.nix ];
domainSuffixExternal = depot.lib.meta.domain;
domainSuffixInternal = "internal.${depot.lib.meta.domain}";
});
default = {};
};
}

97
cluster/services/ways/options/way.nix Normal file
View file

@ -0,0 +1,97 @@
{ config, lib, name, options, ... }:
with lib;
{
options = {
internal = mkOption {
description = "Whether to only make this Way available internally. Will use the internal subdomain.";
type = types.bool;
default = false;
};
name = mkOption {
description = "Domain name to use.";
type = types.str;
default = let
basename = "${name}.${config.domainSuffix}";
in if config.wildcard then "~^(.+)\.${lib.escapeRegex basename}$" else basename;
};
dnsRecord = {
name = mkOption {
description = "DNS record name for this Way.";
type = types.str;
default = if config.wildcard then "^[^_].+\\.${lib.escapeRegex name}" else name;
};
value = mkOption {
description = "DNS record value for this Way.";
type = types.deferredModule;
default = {
consulService = "${name}.ways-proxy";
rewrite.type = lib.mkIf config.wildcard "regex";
};
};
};
target = mkOption {
type = types.str;
};
wildcard = mkOption {
type = types.bool;
default = false;
};
consulService = mkOption {
type = types.str;
};
healthCheckPath = mkOption {
type = types.path;
default = "/.well-known/ways/internal-health-check";
};
useConsul = mkOption {
type = types.bool;
internal = true;
default = false;
};
nginxUpstreamName = mkOption {
type = types.str;
internal = true;
};
domainSuffixInternal = mkOption {
type = types.str;
internal = true;
};
domainSuffixExternal = mkOption {
type = types.str;
internal = true;
};
domainSuffix = mkOption {
type = types.str;
internal = true;
default = if config.internal then config.domainSuffixInternal else config.domainSuffixExternal;
};
extras = mkOption {
description = "Extra configuration to pass to the nginx virtual host submodule.";
type = types.deferredModule;
default = {};
};
};
config = lib.mkMerge [
(lib.mkIf options.consulService.isDefined {
useConsul = true;
nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
target = "http://${options.nginxUpstreamName.value}";
})
];
}