From 8981df6382860b6653681f833b95d05a8d79b8c6 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 22 Nov 2023 23:54:00 +0100 Subject: [PATCH 01/47] packages/grafana: 10.1.5 -> 10.2.0 --- packages/monitoring/grafana/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/monitoring/grafana/default.nix b/packages/monitoring/grafana/default.nix index a496b57..c4c9c75 100644 --- a/packages/monitoring/grafana/default.nix +++ b/packages/monitoring/grafana/default.nix @@ -2,7 +2,7 @@ buildGoModule rec { pname = "grafana"; - version = "10.1.5"; + version = "10.2.0"; excludedPackages = [ "alert_webhook_listener" "clean-swagger" "release_publisher" "slow_proxy" "slow_proxy_mac" "macaron" "devenv" "modowners" ]; @@ -10,15 +10,15 @@ buildGoModule rec { rev = "v${version}"; owner = "grafana"; repo = "grafana"; - hash = "sha256-/caja157OKe9atqZLDzw2oTwhWLNa5DxcgO1iueKow4="; + hash = "sha256-PNKvu7DfVHzBaRGM/Zej0oI5pbi6gPta+ZzVEXXmTsI="; }; srcStatic = fetchurl { url = "https://dl.grafana.com/oss/release/grafana-${version}.linux-amd64.tar.gz"; - hash = "sha256-7LGs/8pbZMEwXHBSPac+guJ3GcYBS3qIRz7JeqZuVQ0="; + hash = "sha256-KE026VWxlJYzRqTqry4h8vm1NIXB7sJUucz+W/s1eoE="; }; - vendorHash = "sha256-KXgGtNHUi+k41GC3Wc5hbJw4k5fxq/p0Je6Q6UZwhtw="; + vendorHash = "sha256-Mybo7ZVP7fwmBwloC3jHJnqPmhbj1DQSwz8T2onkL3Y="; nativeBuildInputs = [ wire ]; From e37587ce806ade87bd7207dcd7e10a54587028f8 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 30 Nov 2023 20:32:14 +0100 Subject: [PATCH 02/47] meta: NixOS 23.11 --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index dacf194..f0b9aaa 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,7 @@ inputs = { systems.url = "github:privatevoid-net/nix-systems-default-linux"; - nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small"; nix-super = { url = "gitlab:max/nix-super?host=git.privatevoid.net"; From ce6a19387a746755c61f8191dd678613d4e48ae8 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 00:59:35 +0100 Subject: [PATCH 03/47] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/8c9fa2545007b49a5db5f650ae91f227672c3877' (2023-11-01) → 'github:hercules-ci/flake-parts/34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5' (2023-12-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d2332963662edffacfddfad59ff4f709dde80ffe' (2023-11-30) → 'github:NixOS/nixpkgs/1bce6a1791a513af2727e5b668b3cd9ba76cb0bf' (2023-11-30) --- flake.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index e62fb5c..b709104 100644 --- a/flake.lock +++ b/flake.lock @@ -233,11 +233,11 @@ ] }, "locked": { - "lastModified": 1698882062, - "narHash": "sha256-HkhafUayIqxXyHH1X8d9RDl1M2CkFgZLjKD3MzabiEo=", + "lastModified": 1701473968, + "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "8c9fa2545007b49a5db5f650ae91f227672c3877", + "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", "type": "github" }, "original": { @@ -474,16 +474,16 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1701362232, - "narHash": "sha256-GVdzxL0lhEadqs3hfRLuj+L1OJFGiL/L7gCcelgBlsw=", + "lastModified": 1701374686, + "narHash": "sha256-xaJPtgvTuUGSPba8p3+ezCJjKnVij77ai8OE2bnTC0E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d2332963662edffacfddfad59ff4f709dde80ffe", + "rev": "1bce6a1791a513af2727e5b668b3cd9ba76cb0bf", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "nixos-23.11-small", "repo": "nixpkgs", "type": "github" } From ce7654740a59755af3f28386f4d9f1067a200df7 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 22 Nov 2023 23:54:50 +0100 Subject: [PATCH 04/47] packages/tempo: 2.2.1 -> 2.3.0 --- packages/patched-derivations.nix | 2 +- packages/sources/sources.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/patched-derivations.nix b/packages/patched-derivations.nix index 8930f66..a3e7e25 100644 --- a/packages/patched-derivations.nix +++ b/packages/patched-derivations.nix @@ -105,7 +105,7 @@ super: rec { ]; }); - tempo = (super.tempo.override { buildGoModule = super.buildGo119Module; }).overrideAttrs (_: { + tempo = (super.tempo.override { buildGoModule = super.buildGo121Module; }).overrideAttrs (_: { version = builtins.substring 1 (-1) pins.tempo.version; src = super.npins.mkSource pins.tempo; subPackages = [ "cmd/tempo" ]; diff --git a/packages/sources/sources.json b/packages/sources/sources.json index 6056121..edfb83e 100644 --- a/packages/sources/sources.json +++ b/packages/sources/sources.json @@ -61,10 +61,10 @@ }, "pre_releases": false, "version_upper_bound": null, - "version": "v2.2.1", - "revision": "77c009c9d315d61207ff3b31c02f94d5749b4bad", - "url": "https://api.github.com/repos/grafana/tempo/tarball/v2.2.1", - "hash": "0biv47mlnsl60nh5z45d3gd4l5avv04l2scmpvyhcrj2fa3abnbh" + "version": "v2.3.0", + "revision": "0b0f48ea2dea728b06ba93bb505fb96b4224fcae", + "url": "https://api.github.com/repos/grafana/tempo/tarball/v2.3.0", + "hash": "08rh22zmx7j5gxsqn4cjr1lg5frmq0bgq8iyvdlgmml5xdbkqj90" } }, "version": 2 From f973ca20842a41b8efe56f8c101160bb65ddd590 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 01:06:20 +0100 Subject: [PATCH 05/47] cluster/services/storage: mkForce garage's StateDirectory --- cluster/services/storage/garage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/services/storage/garage.nix b/cluster/services/storage/garage.nix index b49ec67..6cfd8ed 100644 --- a/cluster/services/storage/garage.nix +++ b/cluster/services/storage/garage.nix @@ -71,7 +71,7 @@ in ProtectSystem = true; User = "garage"; Group = "garage"; - StateDirectory = lib.removePrefix "/var/lib/" cfg.settings.metadata_dir; + StateDirectory = lib.mkForce (lib.removePrefix "/var/lib/" cfg.settings.metadata_dir); }; }; } From d378ff9d066484f48f21e8dff3b2f8a804772dd8 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 01:12:26 +0100 Subject: [PATCH 06/47] modules/fail2ban: switch to submodule style --- modules/fail2ban/default.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/fail2ban/default.nix b/modules/fail2ban/default.nix index e184bd9..22a3eaa 100644 --- a/modules/fail2ban/default.nix +++ b/modules/fail2ban/default.nix @@ -3,11 +3,7 @@ services.fail2ban = { enable = true; banaction = "iptables-multiport[blocktype=DROP]"; - jails.sshd = '' - enabled = true - port = 22 - mode = aggressive - ''; + jails.sshd.settings.mode = "aggressive"; ignoreIP = [ "10.0.0.0/8" depot.reflection.interfaces.primary.addr From fe49607203ce7cec92644e98152ad57def19538e Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 01:14:52 +0100 Subject: [PATCH 07/47] VEGAS/mail: switch fail2ban jails to submodule style --- hosts/VEGAS/services/mail/imap.nix | 4 +--- hosts/VEGAS/services/mail/postfix.nix | 9 ++++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/hosts/VEGAS/services/mail/imap.nix b/hosts/VEGAS/services/mail/imap.nix index b89510f..fc247bc 100644 --- a/hosts/VEGAS/services/mail/imap.nix +++ b/hosts/VEGAS/services/mail/imap.nix @@ -80,7 +80,5 @@ in { systemd.services.dovecot2.serviceConfig.ExecStartPre = [ "${writeLdapConfig}/bin/write-ldap-config" ]; - services.fail2ban.jails.dovecot = '' - enabled = true - ''; + services.fail2ban.jails.dovecot = {}; } diff --git a/hosts/VEGAS/services/mail/postfix.nix b/hosts/VEGAS/services/mail/postfix.nix index 5d679ff..d34b2a1 100644 --- a/hosts/VEGAS/services/mail/postfix.nix +++ b/hosts/VEGAS/services/mail/postfix.nix @@ -93,9 +93,8 @@ in systemd.services.postfix.after = [ "network-online.target" "network-addresses-${interfaces.primary.link}.service" "network-addresses-vstub.service" ]; systemd.services.postfix-setup.after = [ "network-online.target" "network-addresses-${interfaces.primary.link}.service" "network-addresses-vstub.service" ]; - services.fail2ban.jails.postfix = '' - enabled = true - mode = aggressive - findtime = 43200 - ''; + services.fail2ban.jails.postfix.settings = { + mode = "aggressive"; + findtime = "43200"; + }; } From 75042860ace622108c49cb54ba3ec7f7bcc97cae Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 01:20:04 +0100 Subject: [PATCH 08/47] packages/garage: update patchset --- patches/base/garage/print-chill-pills.patch | 53 +++++++++++---------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/patches/base/garage/print-chill-pills.patch b/patches/base/garage/print-chill-pills.patch index a0b4dd9..f2aebf6 100644 --- a/patches/base/garage/print-chill-pills.patch +++ b/patches/base/garage/print-chill-pills.patch @@ -54,8 +54,31 @@ index 11cae4e..ffef3fa 100644 #[derive(Clone)] pub struct Db(pub(crate) Arc); +diff --git a/src/format-table/lib.rs b/src/format-table/lib.rs +index 55252ba..4d8caf1 100644 +--- a/src/format-table/lib.rs ++++ b/src/format-table/lib.rs +@@ -13,6 +13,18 @@ + //! A table to be formatted is a `Vec`, containing one string per line. + //! Table columns in each line are separated by a `\t` character. + ++use std::io::Write; ++ ++macro_rules! print { ++ () => (print!("\n")); ++ ($fmt:expr) => ({ ++ write!(std::io::stdout(), $fmt).unwrap_or(()) ++ }); ++ ($fmt:expr, $($arg:tt)*) => ({ ++ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) ++ }) ++} ++ + /// Format a table and return the result as a string. + pub fn format_table_to_string(data: Vec) -> String { + let data = data diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs -index 0d73588..6bf4ecc 100644 +index cb7a898..97093e6 100644 --- a/src/garage/cli/cmd.rs +++ b/src/garage/cli/cmd.rs @@ -13,6 +13,28 @@ use garage_model::helper::error::Error as HelperError; @@ -111,7 +134,7 @@ index 20813f1..f4baea2 100644 pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> { diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs -index 3884bb9..ef55a66 100644 +index dc5315a..193fd97 100644 --- a/src/garage/cli/layout.rs +++ b/src/garage/cli/layout.rs @@ -8,6 +8,28 @@ use garage_rpc::*; @@ -144,7 +167,7 @@ index 3884bb9..ef55a66 100644 cmd: LayoutOperation, system_rpc_endpoint: &Endpoint, diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs -index 2c6be2f..db6f25d 100644 +index 1140cf2..e4c4d18 100644 --- a/src/garage/cli/util.rs +++ b/src/garage/cli/util.rs @@ -17,6 +17,28 @@ use garage_model::s3::version_table::Version; @@ -177,10 +200,10 @@ index 2c6be2f..db6f25d 100644 println!("List of buckets:"); diff --git a/src/k2v-client/bin/k2v-cli.rs b/src/k2v-client/bin/k2v-cli.rs -index cdd63cc..dfa4df4 100644 +index b9461c8..b9cc148 100644 --- a/src/k2v-client/bin/k2v-cli.rs +++ b/src/k2v-client/bin/k2v-cli.rs -@@ -11,6 +11,28 @@ use rusoto_core::Region; +@@ -10,6 +10,28 @@ use format_table::format_table; use clap::{Parser, Subcommand}; @@ -242,23 +265,3 @@ index 1030e3a..47eca49 100644 /// The layout of the cluster, i.e. the list of roles /// which are assigned to each cluster node #[derive(Clone, Debug, Serialize, Deserialize)] -diff --git a/src/util/formater.rs b/src/util/formater.rs -index 2ea53eb..cc7d8a4 100644 ---- a/src/util/formater.rs -+++ b/src/util/formater.rs -@@ -1,3 +1,15 @@ -+use std::io::Write; -+ -+macro_rules! print { -+ () => (print!("\n")); -+ ($fmt:expr) => ({ -+ write!(std::io::stdout(), $fmt).unwrap_or(()) -+ }); -+ ($fmt:expr, $($arg:tt)*) => ({ -+ write!(std::io::stdout(), $fmt, $($arg)*).unwrap_or(()) -+ }) -+} -+ - pub fn format_table_to_string(data: Vec) -> String { - let data = data - .iter() From 1554d59c7d085751f5369431a8ec5e3b0e5f944e Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 02:48:44 +0100 Subject: [PATCH 09/47] cluster/services/nextcloud: remove enableBrokenCiphersForSSE --- cluster/services/nextcloud/host.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster/services/nextcloud/host.nix b/cluster/services/nextcloud/host.nix index 2e4cd2e..c077c2b 100644 --- a/cluster/services/nextcloud/host.nix +++ b/cluster/services/nextcloud/host.nix @@ -19,7 +19,6 @@ in }; services.nextcloud = { package = pkgs.nextcloud26; - enableBrokenCiphersForSSE = false; enable = true; https = true; hostName = "storage.${depot.lib.meta.domain}"; From 2aeea7be7ffaa2ba03c49073801a834e352448c5 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 02:53:57 +0100 Subject: [PATCH 10/47] modules/deploy-rs-receiver: drop --- modules/deploy-rs-receiver/default.nix | 21 --------------------- modules/part.nix | 2 -- 2 files changed, 23 deletions(-) delete mode 100644 modules/deploy-rs-receiver/default.nix diff --git a/modules/deploy-rs-receiver/default.nix b/modules/deploy-rs-receiver/default.nix deleted file mode 100644 index 006c345..0000000 --- a/modules/deploy-rs-receiver/default.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ - security.sudo.extraRules = [ - { - users = [ "deploy" ]; - commands = [ - "NOPASSWD: /nix/store/*-activate-rs/activate-rs" - "NOPASSWD: /run/current-system/sw/bin/rm /tmp/deploy-rs-canary-*" - ]; - runAs = "root"; - } - ]; - nix.settings.trusted-users = [ "deploy" ]; - users.users.deploy = { - isNormalUser = true; - uid = 1999; - openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMmdWfmAs/0rno8zJlhBFMY2SumnHbTNdZUXJqxgd9ON max@jericho" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL5C7mC5S2gM0K6x0L/jNwAeQYbFSzs16Q73lONUlIkL max@TITAN" - ]; - }; -} diff --git a/modules/part.nix b/modules/part.nix index ae4696a..c8abe8c 100644 --- a/modules/part.nix +++ b/modules/part.nix @@ -10,7 +10,6 @@ in ascensions = ./ascensions; consul-distributed-services = ./consul-distributed-services; consul-service-registry = ./consul-service-registry; - deploy-rs-receiver = ./deploy-rs-receiver; effect-receiver = ./effect-receiver; enterprise = ./enterprise; external-storage = ./external-storage; @@ -50,7 +49,6 @@ in ascensions consul-distributed-services consul-service-registry - deploy-rs-receiver effect-receiver external-storage fail2ban From 75f3a25d3b3d13ce6f35eb3c4ebd4bd377bb2817 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 02:54:45 +0100 Subject: [PATCH 11/47] meta: drop deploy-rs --- flake.nix | 9 --------- packages/patched-inputs.nix | 4 +--- packages/projects.nix | 3 +-- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/flake.nix b/flake.nix index f0b9aaa..c397c2c 100644 --- a/flake.nix +++ b/flake.nix @@ -36,15 +36,6 @@ }; }; - deploy-rs = { - url = "gitlab:max/deploy-rs?host=git.privatevoid.net"; - inputs = { - nixpkgs.follows = "nixpkgs"; - flake-compat.follows = "blank"; - utils.follows = "repin-flake-utils"; - }; - }; - agenix = { url = "github:ryantm/agenix"; inputs.nixpkgs.follows = "nixpkgs"; diff --git a/packages/patched-inputs.nix b/packages/patched-inputs.nix index de82a5f..255563e 100644 --- a/packages/patched-inputs.nix +++ b/packages/patched-inputs.nix @@ -8,8 +8,6 @@ { packages = filters.doFilter filters.packages rec { - inherit (packages.deploy-rs) deploy-rs; - nix-super = packages.nix-super.nix; agenix = packages.agenix.agenix.override { nix = nix-super; }; @@ -17,4 +15,4 @@ hci = packages.hercules-ci-agent.hercules-ci-cli; }; }; -} \ No newline at end of file +} diff --git a/packages/projects.nix b/packages/projects.nix index 9270b24..1f5aab3 100644 --- a/packages/projects.nix +++ b/packages/projects.nix @@ -56,7 +56,6 @@ in { tools = with flakePkgs; [ agenix - deploy-rs dvc graf hci @@ -70,4 +69,4 @@ }; }; }; -} \ No newline at end of file +} From 3a03005445480d81a4206af799adb7b594a7c431 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 02:54:50 +0100 Subject: [PATCH 12/47] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Removed input 'deploy-rs' • Removed input 'deploy-rs/flake-compat' • Removed input 'deploy-rs/nixpkgs' • Removed input 'deploy-rs/utils' --- flake.lock | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/flake.lock b/flake.lock index b709104..be499dd 100644 --- a/flake.lock +++ b/flake.lock @@ -119,34 +119,6 @@ "type": "github" } }, - "deploy-rs": { - "inputs": { - "flake-compat": [ - "blank" - ], - "nixpkgs": [ - "nixpkgs" - ], - "utils": [ - "repin-flake-utils" - ] - }, - "locked": { - "host": "git.privatevoid.net", - "lastModified": 1638903228, - "narHash": "sha256-mEbLD0A9gp159pFtdK4n1Yp2uFSE1T2nOr8BkfwgrC8=", - "owner": "max", - "repo": "deploy-rs", - "rev": "0d11e93f47be21051683e1b38f6b0dcb3f0a71cf", - "type": "gitlab" - }, - "original": { - "host": "git.privatevoid.net", - "owner": "max", - "repo": "deploy-rs", - "type": "gitlab" - } - }, "devshell": { "inputs": { "nixpkgs": [ @@ -513,7 +485,6 @@ "agenix": "agenix", "attic": "attic", "blank": "blank", - "deploy-rs": "deploy-rs", "devshell": "devshell", "drv-parts": "drv-parts", "flake-parts": "flake-parts", From b485a93df4c09143832a207ce4acfdaddb2ca526 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 13:40:51 +0100 Subject: [PATCH 13/47] cluster/services/storage: use consul catalog api for garage discovery --- cluster/services/storage/garage.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster/services/storage/garage.nix b/cluster/services/storage/garage.nix index 6cfd8ed..7783c32 100644 --- a/cluster/services/storage/garage.nix +++ b/cluster/services/storage/garage.nix @@ -39,7 +39,6 @@ in rpc_secret_file = config.age.secrets.garageRpcSecret.path; consul_discovery = { consul_http_addr = "http://127.0.0.1:8500"; - api = "agent"; service_name = "garage-discovery"; }; s3_api = { From ac21ac314a920c0a779bd6003a41b2a8cadefabe Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 19:06:15 +0100 Subject: [PATCH 14/47] packages/kanidm: update patchset --- patches/base/kanidm/unixd-authenticated.patch | 152 ++++++------------ 1 file changed, 50 insertions(+), 102 deletions(-) diff --git a/patches/base/kanidm/unixd-authenticated.patch b/patches/base/kanidm/unixd-authenticated.patch index 8be10f3..3fb2c44 100644 --- a/patches/base/kanidm/unixd-authenticated.patch +++ b/patches/base/kanidm/unixd-authenticated.patch @@ -1,106 +1,54 @@ -diff --git a/unix_integration/src/cache.rs b/unix_integration/src/cache.rs -index d2d442ab8..6c8de0309 100644 ---- a/unix_integration/src/cache.rs -+++ b/unix_integration/src/cache.rs -@@ -34,6 +34,8 @@ enum CacheState { - pub struct CacheLayer { - db: Db, +diff --git a/unix_integration/src/idprovider/kanidm.rs b/unix_integration/src/idprovider/kanidm.rs +index d1b02de0f..599dec6d5 100644 +--- a/unix_integration/src/idprovider/kanidm.rs ++++ b/unix_integration/src/idprovider/kanidm.rs +@@ -2,6 +2,7 @@ use async_trait::async_trait; + use kanidm_client::{ClientError, KanidmClient, StatusCode}; + use kanidm_proto::v1::{OperationError, UnixGroupToken, UnixUserToken}; + use tokio::sync::RwLock; ++use std::env; + + use super::interface::{ + AuthCacheAction, AuthCredHandler, AuthRequest, AuthResult, GroupToken, Id, IdProvider, +@@ -11,12 +12,28 @@ use crate::unix_proto::PamAuthRequest; + + pub struct KanidmProvider { client: RwLock, + auth_name: Option, + auth_password: Option, - state: Mutex, - pam_allow_groups: BTreeSet, - timeout_seconds: u64, -@@ -65,6 +67,8 @@ impl CacheLayer { - timeout_seconds: u64, - // - client: KanidmClient, -+ auth_name: Option, -+ auth_password: Option, - pam_allow_groups: Vec, - default_shell: String, - home_prefix: String, -@@ -91,6 +95,8 @@ impl CacheLayer { - Ok(CacheLayer { - db, + } + + impl KanidmProvider { + pub fn new(client: KanidmClient) -> Self { ++ let env_username: Option; ++ let env_password: Option; ++ match (env::var_os("KANIDM_NAME"), env::var_os("KANIDM_PASSWORD")) { ++ (Some(username), Some(password)) => { ++ env_username = Some(username.into_string().unwrap()); ++ env_password = Some(password.into_string().unwrap()); ++ }, ++ _ => { ++ env_username = None; ++ env_password = None; ++ } ++ } + KanidmProvider { client: RwLock::new(client), -+ auth_name, -+ auth_password, - state: Mutex::new(CacheState::OfflineNextCheck(SystemTime::now())), - timeout_seconds, - pam_allow_groups: pam_allow_groups.into_iter().collect(), -@@ -945,7 +951,11 @@ impl CacheLayer { - false - } - CacheState::OfflineNextCheck(_time) => { -- match self.client.write().await.auth_anonymous().await { -+ let auth_method = match (&self.auth_name, &self.auth_password) { -+ (Some(name), Some(password)) => self.client.write().await.auth_simple_password(name, password).await, -+ _ => self.client.write().await.auth_anonymous().await -+ }; -+ match auth_method { - Ok(_uat) => { - debug!("OfflineNextCheck -> authenticated"); - self.set_cachestate(CacheState::Online).await; -diff --git a/unix_integration/src/daemon.rs b/unix_integration/src/daemon.rs -index e4bf558c6..d6916d851 100644 ---- a/unix_integration/src/daemon.rs -+++ b/unix_integration/src/daemon.rs -@@ -415,6 +415,24 @@ async fn main() -> ExitCode { - .env("KANIDM_CLIENT_CONFIG") - .action(ArgAction::StoreValue), - ) -+ .arg( -+ Arg::new("name") -+ .takes_value(true) -+ .help("Set the name to use to authenticate") -+ .short('D') -+ .long("name") -+ .env("KANIDM_NAME") -+ .action(ArgAction::StoreValue), -+ ) -+ .arg( -+ Arg::new("password") -+ .hide(true) -+ .takes_value(true) -+ .help("Set the password to use to authenticate") -+ .long("password") -+ .env("KANIDM_PASSWORD") -+ .action(ArgAction::StoreValue), -+ ) - .get_matches(); - - if clap_args.get_flag("debug") { -@@ -510,6 +528,10 @@ async fn main() -> ExitCode { - } - } - -+ let auth_username = clap_args.get_one::("name"); -+ -+ let auth_password = clap_args.get_one::("password"); -+ - // setup - let cb = match KanidmClientBuilder::new().read_options_from_optional_config(&cfg_path) { - Ok(v) => v, -@@ -637,6 +659,8 @@ async fn main() -> ExitCode { - cfg.db_path.as_str(), // The sqlite db path - cfg.cache_timeout, - rsclient, -+ auth_username.as_deref().cloned(), -+ auth_password.as_deref().cloned(), - cfg.pam_allowed_login_groups.clone(), - cfg.default_shell.clone(), - cfg.home_prefix.clone(), -diff --git a/unix_integration/tests/cache_layer_test.rs b/unix_integration/tests/cache_layer_test.rs -index cff5e8ba8..a68b35be2 100644 ---- a/unix_integration/tests/cache_layer_test.rs -+++ b/unix_integration/tests/cache_layer_test.rs -@@ -103,6 +103,8 @@ async fn setup_test(fix_fn: Fixture) -> (CacheLayer, KanidmClient) { - "", // The sqlite db path, this is in memory. - 300, - rsclient, -+ None, -+ None, - vec!["allowed_group".to_string()], - DEFAULT_SHELL.to_string(), - DEFAULT_HOME_PREFIX.to_string(), ++ auth_name: env_username, ++ auth_password: env_password, + } + } + } +@@ -73,7 +90,11 @@ impl From for GroupToken { + impl IdProvider for KanidmProvider { + // Needs .read on all types except re-auth. + async fn provider_authenticate(&self) -> Result<(), IdpError> { +- match self.client.write().await.auth_anonymous().await { ++ let auth_method = match (&self.auth_name, &self.auth_password) { ++ (Some(name), Some(password)) => self.client.write().await.auth_simple_password(name, password).await, ++ _ => self.client.write().await.auth_anonymous().await ++ }; ++ match auth_method { + Ok(_uat) => Ok(()), + Err(err) => { + error!(?err, "Provider authentication failed"); From 6f54fa16ca60ea411267ee33391784c46e5cc9b3 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 19:23:53 +0100 Subject: [PATCH 15/47] packages/dvc: drop --- packages/patched-derivations.nix | 47 -- packages/projects.nix | 1 - packages/system-filter.nix | 1 - packages/websites/landing/project.nix | 3 - patches/base/dvc-data/md5-to-sha256.patch | 612 ------------------- patches/base/dvc-objects/md5-to-sha256.patch | 71 --- patches/base/dvc/no-analytics.patch | 267 -------- 7 files changed, 1002 deletions(-) delete mode 100644 patches/base/dvc-data/md5-to-sha256.patch delete mode 100644 patches/base/dvc-objects/md5-to-sha256.patch delete mode 100644 patches/base/dvc/no-analytics.patch diff --git a/packages/patched-derivations.nix b/packages/patched-derivations.nix index a3e7e25..8ebaee1 100644 --- a/packages/patched-derivations.nix +++ b/packages/patched-derivations.nix @@ -2,57 +2,10 @@ let tools = import ./lib/tools.nix; pins = import ./sources; - dvcMd5ToSha256 = old: { - postPatch = (old.postPatch or "") + '' - grep -Rwl md5 | xargs sed -i s/md5/sha256/g - ''; - }; - - dvcYamlToJson = old: { - postPatch = (old.postPatch or "") + '' - grep -Rwl yaml | xargs sed -i s/yaml/json/g - grep -Rwl ruamel.json | xargs sed -i s/ruamel.json/ruamel.yaml/g - ''; - }; in with tools; super: rec { cachix = patch super.cachix "patches/base/cachix"; - dvc = patch (super.dvc.overrideAttrs (old: let - filteredBaseDeps = super.lib.subtractLists [ - super.python3Packages.dvc-data - super.python3Packages.dvc-http - ] old.propagatedBuildInputs; - - baseDeps = filteredBaseDeps ++ [ - dvc-data - dvc-http - ]; - patched = dvcMd5ToSha256 old; - patched' = dvcYamlToJson patched; - in patched' // { - propagatedBuildInputs = with super.python3Packages; baseDeps ++ [ - aiobotocore - boto3 - (s3fs.overrideAttrs (_: { postPatch = '' - substituteInPlace requirements.txt \ - --replace "fsspec==2023.3.0" "fsspec" \ - --replace "aiobotocore~=2.1.0" "aiobotocore" - ''; - })) - ]; - })) "patches/base/dvc"; - - dvc-data = (super.python3Packages.dvc-data.override { - inherit dvc-objects; - }).overrideAttrs dvcMd5ToSha256; - - dvc-http = super.python3Packages.dvc-http.override { - inherit dvc-objects; - }; - - dvc-objects = super.python3Packages.dvc-objects.overrideAttrs dvcMd5ToSha256; - forgejo = patch super.forgejo "patches/base/forgejo"; garage = patch super.garage_0_8 "patches/base/garage"; diff --git a/packages/projects.nix b/packages/projects.nix index 1f5aab3..8312cbd 100644 --- a/packages/projects.nix +++ b/packages/projects.nix @@ -56,7 +56,6 @@ in { tools = with flakePkgs; [ agenix - dvc graf hci npins diff --git a/packages/system-filter.nix b/packages/system-filter.nix index 151b853..42659f1 100644 --- a/packages/system-filter.nix +++ b/packages/system-filter.nix @@ -1,7 +1,6 @@ { packages = { cinny = [ "x86_64-linux" ]; - dvc = [ "x86_64-linux" ]; hci = [ "x86_64-linux" ]; hydra = [ "x86_64-linux" ]; jellyfin = [ "x86_64-linux" ]; diff --git a/packages/websites/landing/project.nix b/packages/websites/landing/project.nix index 9ba2bbc..c356b79 100644 --- a/packages/websites/landing/project.nix +++ b/packages/websites/landing/project.nix @@ -24,9 +24,6 @@ help = pkgs.hugo.meta.description; command = "exec ${pkgs.hugo}/bin/hugo ${hugoArgsStr} \"$@\""; }; - tools = with self'.packages; [ - dvc - ]; }; packages.landing = with pkgs; let diff --git a/patches/base/dvc-data/md5-to-sha256.patch b/patches/base/dvc-data/md5-to-sha256.patch deleted file mode 100644 index fc938c5..0000000 --- a/patches/base/dvc-data/md5-to-sha256.patch +++ /dev/null @@ -1,612 +0,0 @@ -commit d7d093fcb91b0d21faf36dbf62924f23b45abb9b -Author: Max -Date: Sat Dec 17 14:23:59 2022 +0100 - - md5 to sha256 for 2.17.0 - -diff --git a/src/dvc_data/build.py b/src/dvc_data/build.py -index 3656ca5..3837763 100644 ---- a/src/dvc_data/build.py -+++ b/src/dvc_data/build.py -@@ -63,7 +63,7 @@ def _build_file(path, fs, name, odb=None, upload_odb=None, dry_run=False): - state = odb.state if odb else None - meta, hash_info = hash_file(path, fs, name, state=state) - if upload_odb and not dry_run: -- assert odb and name == "md5" -+ assert odb and name == "sha256" - return _upload_file(path, fs, odb, upload_odb) - - oid = hash_info.value -@@ -195,9 +195,9 @@ def _get_staging(odb: "HashFileDB") -> "ReferenceHashFileDB": - def _build_external_tree_info(odb, tree, name): - # NOTE: used only for external outputs. Initial reasoning was to be - # able to validate .dir files right in the workspace (e.g. check s3 -- # etag), but could be dropped for manual validation with regular md5, -+ # etag), but could be dropped for manual validation with regular sha256, - # that would be universal for all clouds. -- assert odb and name != "md5" -+ assert odb and name != "sha256" - - oid = tree.hash_info.value - odb.add(tree.path, tree.fs, oid) -@@ -253,7 +253,7 @@ def build( - **kwargs, - ) - logger.debug("built tree '%s'", obj) -- if name != "md5": -+ if name != "sha256": - obj = _build_external_tree_info(odb, obj, name) - else: - meta, obj = _build_file( -diff --git a/src/dvc_data/cli.py b/src/dvc_data/cli.py -index 2348875..ece639a 100644 ---- a/src/dvc_data/cli.py -+++ b/src/dvc_data/cli.py -@@ -29,8 +29,8 @@ from dvc_data.diff import ROOT - from dvc_data.diff import diff as _diff - from dvc_data.hashfile.db import HashFileDB - from dvc_data.hashfile.hash import algorithms_available --from dvc_data.hashfile.hash import file_md5 as _file_md5 --from dvc_data.hashfile.hash import fobj_md5 as _fobj_md5 -+from dvc_data.hashfile.hash import file_sha256 as _file_sha256 -+from dvc_data.hashfile.hash import fobj_sha256 as _fobj_sha256 - from dvc_data.hashfile.hash_info import HashInfo - from dvc_data.hashfile.obj import HashFile - from dvc_data.hashfile.state import State -@@ -93,7 +93,7 @@ app = Application( - @app.command(name="hash", help="Compute checksum of the file") - def hash_file( - file: Path = file_type, -- name: HashEnum = typer.Option("md5", "-n", "--name"), -+ name: HashEnum = typer.Option("sha256", "-n", "--name"), - progress: bool = typer.Option(False, "--progress", "-p"), - text: Optional[bool] = typer.Option(None, "--text/--binary", "-t/-b"), - ): -@@ -108,9 +108,9 @@ def hash_file( - with callback: - if path == "-": - fobj = callback.wrap_attr(sys.stdin.buffer) -- hash_value = _fobj_md5(fobj, text=text, name=hash_name) -+ hash_value = _fobj_sha256(fobj, text=text, name=hash_name) - else: -- hash_value = _file_md5( -+ hash_value = _file_sha256( - path, name=hash_name, callback=callback, text=text - ) - print(hash_name, hash_value, sep=": ") -@@ -262,7 +262,7 @@ def build( - fs = MemoryFileSystem() - fs.put_file(sys.stdin.buffer, fs_path) - -- object_store, _, obj = _build(odb, fs_path, fs, name="md5") -+ object_store, _, obj = _build(odb, fs_path, fs, name="sha256") - if write: - _transfer( - object_store, -@@ -285,7 +285,7 @@ def ls(oid: str = typer.Argument(..., allow_dash=True)): - odb = get_odb() - oid = from_shortoid(odb, oid) - try: -- tree = Tree.load(odb, HashInfo("md5", oid)) -+ tree = Tree.load(odb, HashInfo("sha256", oid)) - except ObjectFormatError as exc: - typer.echo(exc, err=True) - raise typer.Exit(1) from exc -@@ -454,7 +454,7 @@ def apply_op(odb, obj, application): - ) - - fs = LocalFileSystem() -- _, meta, new_obj = _build(odb, path, fs, "md5") -+ _, meta, new_obj = _build(odb, path, fs, "sha256") - odb.add(path, fs, new_obj.hash_info.value, hardlink=False) - return obj.add(new, meta, new_obj.hash_info) - -diff --git a/src/dvc_data/fs.py b/src/dvc_data/fs.py -index c972981..ac45ad3 100644 ---- a/src/dvc_data/fs.py -+++ b/src/dvc_data/fs.py -@@ -47,7 +47,7 @@ class DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method - if info["type"] == "directory": - raise IsADirectoryError - -- value = info.get("md5") -+ value = info.get("sha256") - if not value: - raise FileNotFoundError - -@@ -142,7 +142,7 @@ class DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method - - def checksum(self, path): - info = self.info(path) -- md5 = info.get("md5") -- if md5: -- return md5 -+ sha256 = info.get("sha256") -+ if sha256: -+ return sha256 - raise NotImplementedError -diff --git a/src/dvc_data/hashfile/hash.py b/src/dvc_data/hashfile/hash.py -index 9bef01d..03f731c 100644 ---- a/src/dvc_data/hashfile/hash.py -+++ b/src/dvc_data/hashfile/hash.py -@@ -42,7 +42,7 @@ class HashStreamFile(io.IOBase): - def __init__( - self, - fobj: BinaryIO, -- hash_name: str = "md5", -+ hash_name: str = "sha256", - text: Optional[bool] = None, - ) -> None: - self.fobj = fobj -@@ -77,11 +77,11 @@ class HashStreamFile(io.IOBase): - return self.hasher.name - - --def fobj_md5( -+def fobj_sha256( - fobj: BinaryIO, - chunk_size: int = 2**20, - text: Optional[bool] = None, -- name="md5", -+ name="sha256", - ) -> str: - # ideally, we want the heuristics to be applied in a similar way, - # regardless of the size of the first chunk, -@@ -95,17 +95,17 @@ def fobj_md5( - return stream.hash_value - - --def file_md5( -+def file_sha256( - fname: "AnyFSPath", - fs: "FileSystem" = localfs, - callback: "Callback" = DEFAULT_CALLBACK, - text: Optional[bool] = None, -- name: str = "md5", -+ name: str = "sha256", - ) -> str: - size = fs.size(fname) or 0 - callback.set_size(size) - with fs.open(fname, "rb") as fobj: -- return fobj_md5(callback.wrap_attr(fobj), text=text, name=name) -+ return fobj_sha256(callback.wrap_attr(fobj), text=text, name=name) - - - def _adapt_info(info: Dict[str, Any], scheme: str) -> Dict[str, Any]: -@@ -139,8 +139,8 @@ def _hash_file( - func = getattr(fs, name) - return str(func(path)), info - -- if name == "md5": -- return file_md5(path, fs, callback=callback), info -+ if name == "sha256": -+ return file_sha256(path, fs, callback=callback), info - raise NotImplementedError - - -@@ -162,7 +162,7 @@ class LargeFileHashingCallback(TqdmCallback): - if self.size and self.size > self.LARGE_FILE_SIZE: - if not self._logged: - logger.info( -- f"Computing md5 for a large file '{self.fname}'. " -+ f"Computing sha256 for a large file '{self.fname}'. " - "This is only done once." - ) - self._logged = True -diff --git a/src/dvc_data/hashfile/utils.py b/src/dvc_data/hashfile/utils.py -index ea2da9c..b1e7726 100644 ---- a/src/dvc_data/hashfile/utils.py -+++ b/src/dvc_data/hashfile/utils.py -@@ -38,7 +38,7 @@ def get_mtime_and_size( - - # We track file changes and moves, which cannot be detected with simply - # max(mtime(f) for f in non_ignored_files) -- hasher = hashlib.md5() -+ hasher = hashlib.sha256() - hasher.update(json.dumps(files_mtimes, sort_keys=True).encode("utf-8")) - mtime = hasher.hexdigest() - return mtime, size -diff --git a/src/dvc_data/objects/tree.py b/src/dvc_data/objects/tree.py -index 4f11fa4..7c8b417 100644 ---- a/src/dvc_data/objects/tree.py -+++ b/src/dvc_data/objects/tree.py -@@ -81,7 +81,7 @@ class Tree(HashFile): - memfs.pipe_file(path, self.as_bytes()) - self.fs = memfs - self.path = path -- _, self.hash_info = hash_file(path, memfs, "md5") -+ _, self.hash_info = hash_file(path, memfs, "sha256") - assert self.hash_info.value - self.hash_info.value += ".dir" - self.oid = self.hash_info.value -diff --git a/tests/hashfile/test_hash.py b/tests/hashfile/test_hash.py -index ca920d8..59bf765 100644 ---- a/tests/hashfile/test_hash.py -+++ b/tests/hashfile/test_hash.py -@@ -2,21 +2,21 @@ from os import fspath - - from dvc_objects.fs import LocalFileSystem - --from dvc_data.hashfile.hash import file_md5 -+from dvc_data.hashfile.hash import file_sha256 - - --def test_file_md5(tmp_path): -+def test_file_sha256(tmp_path): - foo = tmp_path / "foo" - foo.write_text("foo content", encoding="utf8") - - fs = LocalFileSystem() -- assert file_md5(fspath(foo), fs) == file_md5(fspath(foo), fs) -+ assert file_sha256(fspath(foo), fs) == file_sha256(fspath(foo), fs) - - --def test_file_md5_crlf(tmp_path): -+def test_file_sha256_crlf(tmp_path): - fs = LocalFileSystem() - cr = tmp_path / "cr" - crlf = tmp_path / "crlf" - cr.write_bytes(b"a\nb\nc") - crlf.write_bytes(b"a\r\nb\r\nc") -- assert file_md5(fspath(cr), fs) == file_md5(fspath(crlf), fs) -+ assert file_sha256(fspath(cr), fs) == file_sha256(fspath(crlf), fs) -diff --git a/tests/hashfile/test_hash_stream.py b/tests/hashfile/test_hash_stream.py -index a003a29..e67b7c1 100644 ---- a/tests/hashfile/test_hash_stream.py -+++ b/tests/hashfile/test_hash_stream.py -@@ -3,7 +3,7 @@ from os import fspath - import pytest - from dvc_objects.fs import LocalFileSystem - --from dvc_data.hashfile.hash import HashStreamFile, file_md5 -+from dvc_data.hashfile.hash import HashStreamFile, file_sha256 - from dvc_data.hashfile.istextfile import DEFAULT_CHUNK_SIZE, istextfile - - -@@ -23,7 +23,7 @@ def test_hashed_stream_reader(tmp_path): - assert stream_reader.read(1) == b"o" - assert stream_reader.tell() == 3 - -- hex_digest = file_md5(fspath(foo), LocalFileSystem()) -+ hex_digest = file_sha256(fspath(foo), LocalFileSystem()) - assert stream_reader.is_text - assert hex_digest == stream_reader.hash_value - -@@ -46,7 +46,7 @@ def test_hashed_stream_reader_as_chunks(tmp_path): - - assert stream_reader.tell() == actual_size == total_read - -- hex_digest = file_md5(fspath(foo), LocalFileSystem()) -+ hex_digest = file_sha256(fspath(foo), LocalFileSystem()) - assert not stream_reader.is_text - assert hex_digest == stream_reader.hash_value - -@@ -68,7 +68,7 @@ def test_hashed_stream_reader_compatibility(tmp_path, contents): - stream_reader.read(chunk_size) - - local_fs = LocalFileSystem() -- hex_digest = file_md5(fspath(data), local_fs) -+ hex_digest = file_sha256(fspath(data), local_fs) - - assert stream_reader.is_text is istextfile(fspath(data), local_fs) - assert stream_reader.hash_value == hex_digest -diff --git a/tests/hashfile/test_obj.py b/tests/hashfile/test_obj.py -index 01e9fc2..6c47b3c 100644 ---- a/tests/hashfile/test_obj.py -+++ b/tests/hashfile/test_obj.py -@@ -3,7 +3,7 @@ from dvc_data.hashfile.obj import HashFile - - - def test_obj(tmp_upath): -- hash_info = HashInfo("md5", "123456") -+ hash_info = HashInfo("sha256", "123456") - obj = HashFile(tmp_upath, tmp_upath.fs, hash_info) - assert obj.path == tmp_upath - assert obj.fs == tmp_upath.fs -diff --git a/tests/objects/test_tree.py b/tests/objects/test_tree.py -index 6c514ba..611a72f 100644 ---- a/tests/objects/test_tree.py -+++ b/tests/objects/test_tree.py -@@ -13,57 +13,57 @@ from dvc_data.objects.tree import Tree, _merge - ([], {}), - ( - [ -- {"md5": "def", "relpath": "zzz"}, -- {"md5": "123", "relpath": "foo"}, -- {"md5": "abc", "relpath": "aaa"}, -- {"md5": "456", "relpath": "bar"}, -+ {"sha256": "def", "relpath": "zzz"}, -+ {"sha256": "123", "relpath": "foo"}, -+ {"sha256": "abc", "relpath": "aaa"}, -+ {"sha256": "456", "relpath": "bar"}, - ], - { -- ("zzz",): (None, HashInfo("md5", "def")), -- ("foo",): (None, HashInfo("md5", "123")), -- ("bar",): (None, HashInfo("md5", "456")), -- ("aaa",): (None, HashInfo("md5", "abc")), -+ ("zzz",): (None, HashInfo("sha256", "def")), -+ ("foo",): (None, HashInfo("sha256", "123")), -+ ("bar",): (None, HashInfo("sha256", "456")), -+ ("aaa",): (None, HashInfo("sha256", "abc")), - }, - ), - ( - [ -- {"md5": "123", "relpath": "dir/b"}, -- {"md5": "456", "relpath": "dir/z"}, -- {"md5": "789", "relpath": "dir/a"}, -- {"md5": "abc", "relpath": "b"}, -- {"md5": "def", "relpath": "a"}, -- {"md5": "ghi", "relpath": "z"}, -- {"md5": "jkl", "relpath": "dir/subdir/b"}, -- {"md5": "mno", "relpath": "dir/subdir/z"}, -- {"md5": "pqr", "relpath": "dir/subdir/a"}, -+ {"sha256": "123", "relpath": "dir/b"}, -+ {"sha256": "456", "relpath": "dir/z"}, -+ {"sha256": "789", "relpath": "dir/a"}, -+ {"sha256": "abc", "relpath": "b"}, -+ {"sha256": "def", "relpath": "a"}, -+ {"sha256": "ghi", "relpath": "z"}, -+ {"sha256": "jkl", "relpath": "dir/subdir/b"}, -+ {"sha256": "mno", "relpath": "dir/subdir/z"}, -+ {"sha256": "pqr", "relpath": "dir/subdir/a"}, - ], - { - ("dir", "b"): ( - None, -- HashInfo("md5", "123"), -+ HashInfo("sha256", "123"), - ), - ("dir", "z"): ( - None, -- HashInfo("md5", "456"), -+ HashInfo("sha256", "456"), - ), - ("dir", "a"): ( - None, -- HashInfo("md5", "789"), -+ HashInfo("sha256", "789"), - ), -- ("b",): (None, HashInfo("md5", "abc")), -- ("a",): (None, HashInfo("md5", "def")), -- ("z",): (None, HashInfo("md5", "ghi")), -+ ("b",): (None, HashInfo("sha256", "abc")), -+ ("a",): (None, HashInfo("sha256", "def")), -+ ("z",): (None, HashInfo("sha256", "ghi")), - ("dir", "subdir", "b"): ( - None, -- HashInfo("md5", "jkl"), -+ HashInfo("sha256", "jkl"), - ), - ("dir", "subdir", "z"): ( - None, -- HashInfo("md5", "mno"), -+ HashInfo("sha256", "mno"), - ), - ("dir", "subdir", "a"): ( - None, -- HashInfo("md5", "pqr"), -+ HashInfo("sha256", "pqr"), - ), - }, - ), -@@ -81,19 +81,19 @@ def test_list(lst, trie_dict): - ({}, 0), - ( - { -- ("a",): (Meta(size=1), HashInfo("md5", "abc")), -- ("b",): (Meta(size=2), HashInfo("md5", "def")), -- ("c",): (Meta(size=3), HashInfo("md5", "ghi")), -- ("dir", "foo"): (Meta(size=4), HashInfo("md5", "jkl")), -- ("dir", "bar"): (Meta(size=5), HashInfo("md5", "mno")), -- ("dir", "baz"): (Meta(size=6), HashInfo("md5", "pqr")), -+ ("a",): (Meta(size=1), HashInfo("sha256", "abc")), -+ ("b",): (Meta(size=2), HashInfo("sha256", "def")), -+ ("c",): (Meta(size=3), HashInfo("sha256", "ghi")), -+ ("dir", "foo"): (Meta(size=4), HashInfo("sha256", "jkl")), -+ ("dir", "bar"): (Meta(size=5), HashInfo("sha256", "mno")), -+ ("dir", "baz"): (Meta(size=6), HashInfo("sha256", "pqr")), - }, - 6, - ), - ( - { -- ("a",): (Meta(size=1), HashInfo("md5", "abc")), -- ("b",): (Meta(), HashInfo("md5", "def")), -+ ("a",): (Meta(size=1), HashInfo("sha256", "abc")), -+ ("b",): (Meta(), HashInfo("sha256", "def")), - }, - 2, - ), -@@ -110,15 +110,15 @@ def test_nfiles(trie_dict, nfiles): - [ - {}, - { -- ("a",): (None, HashInfo("md5", "abc")), -- ("b",): (None, HashInfo("md5", "def")), -- ("c",): (None, HashInfo("md5", "ghi")), -- ("dir", "foo"): (None, HashInfo("md5", "jkl")), -- ("dir", "bar"): (None, HashInfo("md5", "mno")), -- ("dir", "baz"): (None, HashInfo("md5", "pqr")), -- ("dir", "subdir", "1"): (None, HashInfo("md5", "stu")), -- ("dir", "subdir", "2"): (None, HashInfo("md5", "vwx")), -- ("dir", "subdir", "3"): (None, HashInfo("md5", "yz")), -+ ("a",): (None, HashInfo("sha256", "abc")), -+ ("b",): (None, HashInfo("sha256", "def")), -+ ("c",): (None, HashInfo("sha256", "ghi")), -+ ("dir", "foo"): (None, HashInfo("sha256", "jkl")), -+ ("dir", "bar"): (None, HashInfo("sha256", "mno")), -+ ("dir", "baz"): (None, HashInfo("sha256", "pqr")), -+ ("dir", "subdir", "1"): (None, HashInfo("sha256", "stu")), -+ ("dir", "subdir", "2"): (None, HashInfo("sha256", "vwx")), -+ ("dir", "subdir", "3"): (None, HashInfo("sha256", "yz")), - }, - ], - ) -@@ -135,63 +135,63 @@ def test_items(trie_dict): - [ - ({}, {}, {}, {}), - ( -- {("foo",): HashInfo("md5", "123")}, -+ {("foo",): HashInfo("sha256", "123")}, - { -- ("foo",): HashInfo("md5", "123"), -- ("bar",): HashInfo("md5", "345"), -+ ("foo",): HashInfo("sha256", "123"), -+ ("bar",): HashInfo("sha256", "345"), - }, - { -- ("foo",): HashInfo("md5", "123"), -- ("baz",): HashInfo("md5", "678"), -+ ("foo",): HashInfo("sha256", "123"), -+ ("baz",): HashInfo("sha256", "678"), - }, - { -- ("foo",): HashInfo("md5", "123"), -- ("bar",): HashInfo("md5", "345"), -- ("baz",): HashInfo("md5", "678"), -+ ("foo",): HashInfo("sha256", "123"), -+ ("bar",): HashInfo("sha256", "345"), -+ ("baz",): HashInfo("sha256", "678"), - }, - ), - ( - { -- ("common",): HashInfo("md5", "123"), -- ("subdir", "foo"): HashInfo("md5", "345"), -+ ("common",): HashInfo("sha256", "123"), -+ ("subdir", "foo"): HashInfo("sha256", "345"), - }, - { -- ("common",): HashInfo("md5", "123"), -- ("subdir", "foo"): HashInfo("md5", "345"), -- ("subdir", "bar"): HashInfo("md5", "678"), -+ ("common",): HashInfo("sha256", "123"), -+ ("subdir", "foo"): HashInfo("sha256", "345"), -+ ("subdir", "bar"): HashInfo("sha256", "678"), - }, - { -- ("common",): HashInfo("md5", "123"), -- ("subdir", "foo"): HashInfo("md5", "345"), -- ("subdir", "baz"): HashInfo("md5", "91011"), -+ ("common",): HashInfo("sha256", "123"), -+ ("subdir", "foo"): HashInfo("sha256", "345"), -+ ("subdir", "baz"): HashInfo("sha256", "91011"), - }, - { -- ("common",): HashInfo("md5", "123"), -- ("subdir", "foo"): HashInfo("md5", "345"), -- ("subdir", "bar"): HashInfo("md5", "678"), -- ("subdir", "baz"): HashInfo("md5", "91011"), -+ ("common",): HashInfo("sha256", "123"), -+ ("subdir", "foo"): HashInfo("sha256", "345"), -+ ("subdir", "bar"): HashInfo("sha256", "678"), -+ ("subdir", "baz"): HashInfo("sha256", "91011"), - }, - ), - ( - {}, -- {("foo",): HashInfo("md5", "123")}, -- {("bar",): HashInfo("md5", "456")}, -+ {("foo",): HashInfo("sha256", "123")}, -+ {("bar",): HashInfo("sha256", "456")}, - { -- ("foo",): HashInfo("md5", "123"), -- ("bar",): HashInfo("md5", "456"), -+ ("foo",): HashInfo("sha256", "123"), -+ ("bar",): HashInfo("sha256", "456"), - }, - ), - ( - {}, - {}, -- {("bar",): HashInfo("md5", "123")}, -- {("bar",): HashInfo("md5", "123")}, -+ {("bar",): HashInfo("sha256", "123")}, -+ {("bar",): HashInfo("sha256", "123")}, - ), - ( - {}, -- {("bar",): HashInfo("md5", "123")}, -+ {("bar",): HashInfo("sha256", "123")}, - {}, -- {("bar",): HashInfo("md5", "123")}, -+ {("bar",): HashInfo("sha256", "123")}, - ), - ], - ) -diff --git a/tests/test_index.py b/tests/test_index.py -index c6404fa..635bf66 100644 ---- a/tests/test_index.py -+++ b/tests/test_index.py -@@ -17,8 +17,8 @@ def odb(tmp_upath_factory, as_filesystem): - - data = tmp_upath_factory.mktemp() / "data.dir" - data.write_bytes( -- b'[{"md5": "c157a79031e1c40f85931829bc5fc552", "relpath": "bar"}, ' -- b'{"md5": "258622b1688250cb619f3c9ccaefb7eb", "relpath": "baz"}]' -+ b'[{"sha256": "c157a79031e1c40f85931829bc5fc552", "relpath": "bar"}, ' -+ b'{"sha256": "258622b1688250cb619f3c9ccaefb7eb", "relpath": "baz"}]' - ) - - bar = tmp_upath_factory.mktemp() / "bar" -@@ -46,13 +46,13 @@ def test_fs(tmp_upath, odb, as_filesystem): - ("foo",): DataIndexEntry( - odb=odb, - hash_info=HashInfo( -- name="md5", value="d3b07384d113edec49eaa6238ad5ff00" -+ name="sha256", value="d3b07384d113edec49eaa6238ad5ff00" - ), - ), - ("data",): DataIndexEntry( - odb=odb, - hash_info=HashInfo( -- name="md5", -+ name="sha256", - value="1f69c66028c35037e8bf67e5bc4ceb6a.dir", - ), - ), -@@ -80,22 +80,22 @@ def test_build(tmp_upath, odb, as_filesystem): - }, - ) - build(index, tmp_upath, as_filesystem(tmp_upath.fs)) -- assert index[("foo",)].hash_info.name == "md5" -+ assert index[("foo",)].hash_info.name == "sha256" - assert ( - index[("foo",)].hash_info.value == "d3b07384d113edec49eaa6238ad5ff00" - ) - assert index[("foo",)].odb == odb -- assert index[("data",)].hash_info.name == "md5" -+ assert index[("data",)].hash_info.name == "sha256" - assert ( - index[("data",)].hash_info.value - == "1f69c66028c35037e8bf67e5bc4ceb6a.dir" - ) -- assert index[("data", "bar")].hash_info.name == "md5" -+ assert index[("data", "bar")].hash_info.name == "sha256" - assert ( - index[("data", "bar")].hash_info.value - == "c157a79031e1c40f85931829bc5fc552" - ) -- assert index[("data", "baz")].hash_info.name == "md5" -+ assert index[("data", "baz")].hash_info.name == "sha256" - assert ( - index[("data", "baz")].hash_info.value - == "258622b1688250cb619f3c9ccaefb7eb" -@@ -108,13 +108,13 @@ def test_checkout(tmp_upath, odb, as_filesystem): - ("foo",): DataIndexEntry( - odb=odb, - hash_info=HashInfo( -- name="md5", value="d3b07384d113edec49eaa6238ad5ff00" -+ name="sha256", value="d3b07384d113edec49eaa6238ad5ff00" - ), - ), - ("data",): DataIndexEntry( - odb=odb, - hash_info=HashInfo( -- name="md5", -+ name="sha256", - value="1f69c66028c35037e8bf67e5bc4ceb6a.dir", - ), - ), diff --git a/patches/base/dvc-objects/md5-to-sha256.patch b/patches/base/dvc-objects/md5-to-sha256.patch deleted file mode 100644 index bc3b532..0000000 --- a/patches/base/dvc-objects/md5-to-sha256.patch +++ /dev/null @@ -1,71 +0,0 @@ -commit 2065fc148ce77be68c95a81a05391e1bb35da79d -Author: Max -Date: Sat Dec 17 14:35:20 2022 +0100 - - md5 to sha256 for 2.17.0 - -diff --git a/src/dvc_objects/db.py b/src/dvc_objects/db.py -index 0f0ab16..3b87fdb 100644 ---- a/src/dvc_objects/db.py -+++ b/src/dvc_objects/db.py -@@ -229,7 +229,7 @@ class ObjectDB: - returned. - - NOTE: For large remotes the list of oids will be very -- big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list) -+ big(e.g. 100M entries, sha256 for each is 32 bytes, so ~3200Mb list) - and we don't really need all of it at the same time, so it makes - sense to use a generator to gradually iterate over it, without - keeping all of it in memory. -diff --git a/src/dvc_objects/fs/__init__.py b/src/dvc_objects/fs/__init__.py -index d236fdc..74db3fe 100644 ---- a/src/dvc_objects/fs/__init__.py -+++ b/src/dvc_objects/fs/__init__.py -@@ -62,7 +62,7 @@ def get_fs_cls(remote_conf, cls=None, scheme=None): - - def as_filesystem( - fs: "AbstractFileSystem", -- checksum: str = "md5", -+ checksum: str = "sha256", - object_based: bool = False, - **fs_args, - ) -> "FileSystem": -diff --git a/src/dvc_objects/fs/implementations/local.py b/src/dvc_objects/fs/implementations/local.py -index 7f888ec..3e1a61a 100644 ---- a/src/dvc_objects/fs/implementations/local.py -+++ b/src/dvc_objects/fs/implementations/local.py -@@ -167,7 +167,7 @@ class LocalFileSystem(FileSystem): - sep = os.sep - - protocol = "local" -- PARAM_CHECKSUM = "md5" -+ PARAM_CHECKSUM = "sha256" - PARAM_PATH = "path" - TRAVERSE_PREFIX_LEN = 2 - -diff --git a/src/dvc_objects/fs/implementations/memory.py b/src/dvc_objects/fs/implementations/memory.py -index 97702cb..c5b5ad7 100644 ---- a/src/dvc_objects/fs/implementations/memory.py -+++ b/src/dvc_objects/fs/implementations/memory.py -@@ -3,7 +3,7 @@ from ..base import FileSystem - - class MemoryFileSystem(FileSystem): # pylint:disable=abstract-method - protocol = "memory" -- PARAM_CHECKSUM = "md5" -+ PARAM_CHECKSUM = "sha256" - - def __init__(self, global_store=True, trie_based=False, fs=None, **kwargs): - super().__init__(fs=fs, **kwargs) -diff --git a/src/dvc_objects/fs/implementations/ssh.py b/src/dvc_objects/fs/implementations/ssh.py -index 8b93faf..8aed5e4 100644 ---- a/src/dvc_objects/fs/implementations/ssh.py -+++ b/src/dvc_objects/fs/implementations/ssh.py -@@ -24,7 +24,7 @@ def ask_password(host, user, port): - class SSHFileSystem(FileSystem): - protocol = "ssh" - REQUIRES = {"sshfs": "sshfs"} -- PARAM_CHECKSUM = "md5" -+ PARAM_CHECKSUM = "sha256" - - @classmethod - def _strip_protocol(cls, path: str) -> str: diff --git a/patches/base/dvc/no-analytics.patch b/patches/base/dvc/no-analytics.patch deleted file mode 100644 index 817fab9..0000000 --- a/patches/base/dvc/no-analytics.patch +++ /dev/null @@ -1,267 +0,0 @@ -diff --git a/dvc/analytics.py b/dvc/analytics.py -deleted file mode 100644 -index 6e3dc91..0000000 ---- a/dvc/analytics.py -+++ /dev/null -@@ -1,156 +0,0 @@ --import json --import logging --import os -- --from .env import DVC_NO_ANALYTICS -- --logger = logging.getLogger(__name__) -- -- --def collect_and_send_report(args=None, return_code=None): -- """ -- Collect information from the runtime/environment and the command -- being executed into a report and send it over the network. -- -- To prevent analytics from blocking the execution of the main thread, -- sending the report is done in a separate process. -- -- The inter-process communication happens through a file containing the -- report as a JSON, where the _collector_ generates it and the _sender_ -- removes it after sending it. -- """ -- import tempfile -- -- from dvc.daemon import daemon -- -- report = {} -- -- # Include command execution information on the report only when available. -- if args and hasattr(args, "func"): -- report.update({"cmd_class": args.func.__name__}) -- -- if return_code is not None: -- report.update({"cmd_return_code": return_code}) -- -- with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: -- json.dump(report, fobj) -- daemon(["analytics", fobj.name]) -- -- --def is_enabled(): -- from dvc.config import Config, to_bool -- from dvc.utils import env2bool -- -- if env2bool("DVC_TEST"): -- return False -- -- enabled = not os.getenv(DVC_NO_ANALYTICS) -- if enabled: -- enabled = to_bool( -- Config.from_cwd(validate=False).get("core", {}).get("analytics", "true") -- ) -- -- logger.debug("Analytics is %sabled.", "en" if enabled else "dis") -- -- return enabled -- -- --def send(path): -- """ -- Side effect: Removes the report after sending it. -- -- The report is generated and stored in a temporary file, see: -- `collect_and_send_report`. Sending happens on another process, -- thus, the need of removing such file afterwards. -- """ -- import requests -- -- url = "https://analytics.dvc.org" -- headers = {"content-type": "application/json"} -- -- with open(path, encoding="utf-8") as fobj: -- report = json.load(fobj) -- -- report.update(_runtime_info()) -- -- try: -- requests.post(url, json=report, headers=headers, timeout=5) -- except requests.exceptions.RequestException: -- logger.debug("failed to send analytics report", exc_info=True) -- -- os.remove(path) -- -- --def _scm_in_use(): -- from dvc.exceptions import NotDvcRepoError -- from dvc.repo import Repo -- from dvc.scm import NoSCM -- -- from .scm import SCM, SCMError -- -- try: -- scm = SCM(root_dir=Repo.find_root()) -- return type(scm).__name__ -- except SCMError: -- return NoSCM.__name__ -- except NotDvcRepoError: -- pass -- -- --def _runtime_info(): -- """ -- Gather information from the environment where DVC runs to fill a report. -- """ -- from iterative_telemetry import _generate_ci_id, find_or_create_user_id -- -- from dvc import __version__ -- from dvc.utils import is_binary -- -- ci_id = _generate_ci_id() -- if ci_id: -- group_id, user_id = ci_id -- else: -- group_id, user_id = None, find_or_create_user_id() -- -- return { -- "dvc_version": __version__, -- "is_binary": is_binary(), -- "scm_class": _scm_in_use(), -- "system_info": _system_info(), -- "user_id": user_id, -- "group_id": group_id, -- } -- -- --def _system_info(): -- import platform -- import sys -- -- import distro -- -- system = platform.system() -- -- if system == "Windows": -- version = sys.getwindowsversion() # type: ignore[attr-defined] -- -- return { -- "os": "windows", -- "windows_version_build": version.build, -- "windows_version_major": version.major, -- "windows_version_minor": version.minor, -- "windows_version_service_pack": version.service_pack, -- } -- -- if system == "Darwin": -- return {"os": "mac", "mac_version": platform.mac_ver()[0]} -- -- if system == "Linux": -- return { -- "os": "linux", -- "linux_distro": distro.id(), -- "linux_distro_like": distro.like(), -- "linux_distro_version": distro.version(), -- } -- -- # We don't collect data for any other system. -- raise NotImplementedError -diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py -index 274b564..b601d84 100644 ---- a/dvc/cli/__init__.py -+++ b/dvc/cli/__init__.py -@@ -236,11 +236,6 @@ def main(argv=None): # noqa: C901, PLR0912, PLR0915 - ret = _log_exceptions(exc) or 255 - - try: -- from dvc import analytics -- -- if analytics.is_enabled(): -- analytics.collect_and_send_report(args, ret) -- - return ret - finally: - logger.setLevel(outer_log_level) -diff --git a/dvc/commands/daemon.py b/dvc/commands/daemon.py -index 35d6e90..d5a7b6e 100644 ---- a/dvc/commands/daemon.py -+++ b/dvc/commands/daemon.py -@@ -26,15 +26,6 @@ class CmdDaemonUpdater(CmdDaemonBase): - return 0 - - --class CmdDaemonAnalytics(CmdDaemonBase): -- def run(self): -- from dvc import analytics -- -- analytics.send(self.args.target) -- -- return 0 -- -- - def add_parser(subparsers, parent_parser): - DAEMON_HELP = "Service daemon." - daemon_parser = subparsers.add_parser( -@@ -59,15 +50,3 @@ def add_parser(subparsers, parent_parser): - help=DAEMON_UPDATER_HELP, - ) - daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) -- -- DAEMON_ANALYTICS_HELP = "Send dvc usage analytics." -- daemon_analytics_parser = daemon_subparsers.add_parser( -- "analytics", -- parents=[parent_parser], -- description=DAEMON_ANALYTICS_HELP, -- help=DAEMON_ANALYTICS_HELP, -- ) -- daemon_analytics_parser.add_argument( -- "target", help="Analytics file." -- ).complete = completion.FILE -- daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) -diff --git a/dvc/commands/init.py b/dvc/commands/init.py -index ca44919..05730aa 100644 ---- a/dvc/commands/init.py -+++ b/dvc/commands/init.py -@@ -3,7 +3,6 @@ import logging - - import colorama - --from dvc import analytics - from dvc.cli.command import CmdBaseNoRepo - from dvc.cli.utils import append_doc_link - from dvc.utils import boxify -@@ -15,16 +14,6 @@ logger = logging.getLogger(__name__) - def _welcome_message(): - from dvc.ui import ui - -- if analytics.is_enabled(): -- ui.write( -- boxify( -- "DVC has enabled anonymous aggregate usage analytics.\n" -- "Read the analytics documentation (and how to opt-out) here:\n" -- + fmt_link("https://dvc.org/doc/user-guide/analytics"), -- border_color="red", -- ) -- ) -- - msg = ( - "{yellow}What's next?{nc}\n" - "{yellow}------------{nc}\n" -diff --git a/dvc/config_schema.py b/dvc/config_schema.py -index 2e36e90..3d9e402 100644 ---- a/dvc/config_schema.py -+++ b/dvc/config_schema.py -@@ -144,7 +144,6 @@ SCHEMA = { - "remote": Lower, - "checksum_jobs": All(Coerce(int), Range(1)), - Optional("interactive", default=False): Bool, -- Optional("analytics", default=True): Bool, - Optional("hardlink_lock", default=False): Bool, - Optional("no_scm", default=False): Bool, - Optional("autostage", default=False): Bool, -diff --git a/dvc/env.py b/dvc/env.py -index 081ec9d..06c1332 100644 ---- a/dvc/env.py -+++ b/dvc/env.py -@@ -7,7 +7,6 @@ DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE" - DVC_EXP_NAME = "DVC_EXP_NAME" - DVC_GLOBAL_CONFIG_DIR = "DVC_GLOBAL_CONFIG_DIR" - DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY" --DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS" - DVC_PAGER = "DVC_PAGER" - DVC_ROOT = "DVC_ROOT" - DVC_SHOW_TRACEBACK = "DVC_SHOW_TRACEBACK" From 28d2e668f721d3148f34b4895b8c79612bc89d58 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 19:27:36 +0100 Subject: [PATCH 16/47] VEGAS/api: WEBHOOK_URL -> webhookUrl --- hosts/VEGAS/services/api/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/VEGAS/services/api/default.nix b/hosts/VEGAS/services/api/default.nix index fec66bb..ca7d25c 100644 --- a/hosts/VEGAS/services/api/default.nix +++ b/hosts/VEGAS/services/api/default.nix @@ -13,6 +13,7 @@ in services.n8n = { enable = true; + webhookUrl = "https://${apiAddr}"; settings = { inherit (config.links.api) port; }; @@ -22,7 +23,6 @@ in N8N_LISTEN_ADDRESS = "127.0.0.1"; N8N_ENDPOINT_WEBHOOK = "api"; N8N_ENDPOINT_WEBHOOK_TEST = "test"; - WEBHOOK_URL = "https://${apiAddr}"; }; services.nginx.virtualHosts."${apiAddr}" = lib.recursiveUpdate proxy { From 8ebbd3e3b5bc2aacba534836975945466535f30a Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 19:32:42 +0100 Subject: [PATCH 17/47] packages: vendorSha256 -> vendorHash --- packages/networking/hyprspace/project.nix | 2 +- packages/networking/ipfs-cluster/project.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/networking/hyprspace/project.nix b/packages/networking/hyprspace/project.nix index abbd21d..7759f4e 100644 --- a/packages/networking/hyprspace/project.nix +++ b/packages/networking/hyprspace/project.nix @@ -30,7 +30,7 @@ ]); }; - vendorSha256 = "sha256-VBCgFbJixBh+pKfYGJVapHqWBpUFfvjl1cwOER2Li6Y="; + vendorHash = "sha256-VBCgFbJixBh+pKfYGJVapHqWBpUFfvjl1cwOER2Li6Y="; ldflags = [ "-s" "-w" "-X github.com/hyprspace/hyprspace/cli.appVersion=${version}" ]; diff --git a/packages/networking/ipfs-cluster/project.nix b/packages/networking/ipfs-cluster/project.nix index 9e89fd2..be54698 100644 --- a/packages/networking/ipfs-cluster/project.nix +++ b/packages/networking/ipfs-cluster/project.nix @@ -43,7 +43,7 @@ ]); }; - vendorSha256 = "sha256-EpZQ7br+ChoAGIj0g6pdpWvFeOFOn2i+6YRBgtzoO+A="; + vendorHash = "sha256-EpZQ7br+ChoAGIj0g6pdpWvFeOFOn2i+6YRBgtzoO+A="; doCheck = false; From b0bff5c9b027a5b927ec79134ade41bd8f81ea73 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 2 Dec 2023 23:44:11 +0100 Subject: [PATCH 18/47] packages/acme-dns: init patched --- packages/patched-derivations.nix | 2 + patches/base/acme-dns/direct.patch | 182 ++++++++++++++++++ .../acme-dns/do-not-lowercase-records.patch | 13 ++ 3 files changed, 197 insertions(+) create mode 100644 patches/base/acme-dns/direct.patch create mode 100644 patches/base/acme-dns/do-not-lowercase-records.patch diff --git a/packages/patched-derivations.nix b/packages/patched-derivations.nix index 8ebaee1..cb0998b 100644 --- a/packages/patched-derivations.nix +++ b/packages/patched-derivations.nix @@ -4,6 +4,8 @@ let in with tools; super: rec { + acme-dns = patch super.acme-dns "patches/base/acme-dns"; + cachix = patch super.cachix "patches/base/cachix"; forgejo = patch super.forgejo "patches/base/forgejo"; diff --git a/patches/base/acme-dns/direct.patch b/patches/base/acme-dns/direct.patch new file mode 100644 index 0000000..0b88923 --- /dev/null +++ b/patches/base/acme-dns/direct.patch @@ -0,0 +1,182 @@ +diff --git a/acmetxt.go b/acmetxt.go +index 63454a6..e7ba7ea 100644 +--- a/acmetxt.go ++++ b/acmetxt.go +@@ -12,6 +12,7 @@ import ( + type ACMETxt struct { + Username uuid.UUID + Password string ++ Direct bool + ACMETxtPost + AllowFrom cidrslice + } +diff --git a/api.go b/api.go +index 864256c..beb16c4 100644 +--- a/api.go ++++ b/api.go +@@ -82,15 +82,15 @@ func webUpdatePost(w http.ResponseWriter, r *http.Request, _ httprouter.Params) + // NOTE: An invalid subdomain should not happen - the auth handler should + // reject POSTs with an invalid subdomain before this handler. Reject any + // invalid subdomains anyway as a matter of caution. +- if !validSubdomain(a.Subdomain) { ++ if !a.Direct && !validSubdomain(a.Subdomain) { + log.WithFields(log.Fields{"error": "subdomain", "subdomain": a.Subdomain, "txt": a.Value}).Debug("Bad update data") + updStatus = http.StatusBadRequest + upd = jsonError("bad_subdomain") +- } else if !validTXT(a.Value) { ++ } else if !a.Direct && !validTXT(a.Value) { + log.WithFields(log.Fields{"error": "txt", "subdomain": a.Subdomain, "txt": a.Value}).Debug("Bad update data") + updStatus = http.StatusBadRequest + upd = jsonError("bad_txt") +- } else if validSubdomain(a.Subdomain) && validTXT(a.Value) { ++ } else if a.Direct || (validSubdomain(a.Subdomain) && validTXT(a.Value)) { + err := DB.Update(a.ACMETxtPost) + if err != nil { + log.WithFields(log.Fields{"error": err.Error()}).Debug("Error while trying to update record") +diff --git a/auth.go b/auth.go +index c09f8b4..c91214d 100644 +--- a/auth.go ++++ b/auth.go +@@ -6,6 +6,7 @@ import ( + "fmt" + "net" + "net/http" ++ "os" + + "github.com/julienschmidt/httprouter" + log "github.com/sirupsen/logrus" +@@ -20,6 +21,18 @@ const ACMETxtKey key = 0 + func Auth(update httprouter.Handle) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + postData := ACMETxt{} ++ directKey := r.Header.Get("X-Direct-Key") ++ if directKey != "" && directKey == os.Getenv("ACME_DNS_DIRECT_STATIC_KEY") { ++ dec := json.NewDecoder(r.Body) ++ err := dec.Decode(&postData) ++ if err != nil { ++ log.WithFields(log.Fields{"error": "json_error", "string": err.Error()}).Error("Decode error") ++ } ++ postData.Direct = true ++ ctx := context.WithValue(r.Context(), ACMETxtKey, postData) ++ update(w, r.WithContext(ctx), p) ++ return ++ } + userOK := false + user, err := getUserFromRequest(r) + if err == nil { +diff --git a/db.go b/db.go +index 3534728..4a389ac 100644 +--- a/db.go ++++ b/db.go +@@ -35,7 +35,7 @@ var userTable = ` + + var txtTable = ` + CREATE TABLE IF NOT EXISTS txt( +- Subdomain TEXT NOT NULL, ++ Subdomain TEXT NOT NULL PRIMARY KEY, + Value TEXT NOT NULL DEFAULT '', + LastUpdate INT + );` +@@ -43,7 +43,7 @@ var txtTable = ` + var txtTablePG = ` + CREATE TABLE IF NOT EXISTS txt( + rowid SERIAL, +- Subdomain TEXT NOT NULL, ++ Subdomain TEXT NOT NULL PRIMARY KEY, + Value TEXT NOT NULL DEFAULT '', + LastUpdate INT + );` +@@ -250,7 +250,6 @@ func (d *acmedb) GetByUsername(u uuid.UUID) (ACMETxt, error) { + func (d *acmedb) GetTXTForDomain(domain string) ([]string, error) { + d.Lock() + defer d.Unlock() +- domain = sanitizeString(domain) + var txts []string + getSQL := ` + SELECT Value FROM txt WHERE Subdomain=$1 LIMIT 2 +@@ -289,9 +288,11 @@ func (d *acmedb) Update(a ACMETxtPost) error { + timenow := time.Now().Unix() + + updSQL := ` +- UPDATE txt SET Value=$1, LastUpdate=$2 +- WHERE rowid=( +- SELECT rowid FROM txt WHERE Subdomain=$3 ORDER BY LastUpdate LIMIT 1) ++ INSERT INTO txt (Value, LastUpdate, Subdomain) ++ VALUES ($1, $2, $3) ++ ON CONFLICT (Subdomain) DO UPDATE SET ++ Value = excluded.Value, ++ LastUpdate = excluded.LastUpdate; + ` + if Config.Database.Engine == "sqlite3" { + updSQL = getSQLiteStmt(updSQL) +diff --git a/db_test.go b/db_test.go +index beca9c1..b775cf4 100644 +--- a/db_test.go ++++ b/db_test.go +@@ -251,19 +251,12 @@ func TestGetTXTForDomain(t *testing.T) { + t.Errorf("No rows returned for GetTXTForDomain [%s]", reg.Subdomain) + } + +- var val1found = false + var val2found = false + for _, v := range regDomainSlice { +- if v == txtval1 { +- val1found = true +- } + if v == txtval2 { + val2found = true + } + } +- if !val1found { +- t.Errorf("No TXT value found for val1") +- } + if !val2found { + t.Errorf("No TXT value found for val2") + } +diff --git a/dns.go b/dns.go +index 9a3b06b..6e8b3d8 100644 +--- a/dns.go ++++ b/dns.go +@@ -195,16 +195,12 @@ func (d *DNSServer) answer(q dns.Question) ([]dns.RR, int, bool, error) { + var err error + var txtRRs []dns.RR + var authoritative = d.isAuthoritative(q) +- if !d.isOwnChallenge(q.Name) && !d.answeringForDomain(q.Name) { ++ if !d.answeringForDomain(q.Name) { + rcode = dns.RcodeNameError + } + r, _ := d.getRecord(q) + if q.Qtype == dns.TypeTXT { +- if d.isOwnChallenge(q.Name) { +- txtRRs, err = d.answerOwnChallenge(q) +- } else { +- txtRRs, err = d.answerTXT(q) +- } ++ txtRRs, err = d.answerTXT(q) + if err == nil { + r = append(r, txtRRs...) + } +@@ -219,7 +215,7 @@ func (d *DNSServer) answer(q dns.Question) ([]dns.RR, int, bool, error) { + + func (d *DNSServer) answerTXT(q dns.Question) ([]dns.RR, error) { + var ra []dns.RR +- subdomain := sanitizeDomainQuestion(q.Name) ++ subdomain, _ := strings.CutSuffix(sanitizeDomainQuestion(q.Name), "."+d.Domain) + atxt, err := d.DB.GetTXTForDomain(subdomain) + if err != nil { + log.WithFields(log.Fields{"error": err.Error()}).Debug("Error while trying to get record") +diff --git a/util.go b/util.go +index 163683d..007907d 100644 +--- a/util.go ++++ b/util.go +@@ -83,6 +83,10 @@ func generatePassword(length int) string { + + func sanitizeDomainQuestion(d string) string { + dom := strings.ToLower(d) ++ // HACK ++ if strings.HasPrefix(dom, "_acme-challenge") { ++ return dom ++ } + firstDot := strings.Index(d, ".") + if firstDot > 0 { + dom = dom[0:firstDot] diff --git a/patches/base/acme-dns/do-not-lowercase-records.patch b/patches/base/acme-dns/do-not-lowercase-records.patch new file mode 100644 index 0000000..7251504 --- /dev/null +++ b/patches/base/acme-dns/do-not-lowercase-records.patch @@ -0,0 +1,13 @@ +diff --git a/dns.go b/dns.go +index a01fb9c..9a3b06b 100644 +--- a/dns.go ++++ b/dns.go +@@ -51,7 +51,7 @@ func (d *DNSServer) Start(errorChannel chan error) { + // ParseRecords parses a slice of DNS record string + func (d *DNSServer) ParseRecords(config DNSConfig) { + for _, v := range config.General.StaticRecords { +- rr, err := dns.NewRR(strings.ToLower(v)) ++ rr, err := dns.NewRR(v) + if err != nil { + log.WithFields(log.Fields{"error": err.Error(), "rr": v}).Warning("Could not parse RR from config") + continue From eaa4bdb4497fdba811c8d3e24dbb950aa9231984 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 01:43:26 +0100 Subject: [PATCH 19/47] cluster/services/dns: support TXT records --- cluster/services/dns/options.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/services/dns/options.nix b/cluster/services/dns/options.nix index 597db7d..fc93a8e 100644 --- a/cluster/services/dns/options.nix +++ b/cluster/services/dns/options.nix @@ -19,7 +19,7 @@ let }; type = mkOption { - type = types.enum [ "A" "CNAME" "AAAA" "NS" "MX" "SOA" ]; + type = types.enum [ "A" "CNAME" "AAAA" "NS" "MX" "SOA" "TXT" ]; default = "A"; }; target = mkOption { From 2a9fdfa4f9748d9a62a0684e3cbbbfb1ee58a44c Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 16:30:16 +0100 Subject: [PATCH 20/47] cluster/services/dns: switch to acme-dns, host static records --- .../services/dns/acme-dns-db-credentials.age | 16 +++ cluster/services/dns/acme-dns-direct-key.age | 21 ++++ cluster/services/dns/admin.nix | 109 ------------------ cluster/services/dns/authoritative.nix | 97 ++++++++++------ cluster/services/dns/coredns.nix | 31 +++-- cluster/services/dns/default.nix | 29 +++-- .../services/dns/pdns-admin-oidc-secrets.age | Bin 539 -> 0 bytes cluster/services/dns/pdns-admin-salt.age | 11 -- cluster/services/dns/pdns-admin-secret.age | 12 -- cluster/services/dns/pdns-api-key.age | Bin 1279 -> 0 bytes cluster/services/dns/pdns-db-credentials.age | 20 ---- secrets.nix | 7 +- 12 files changed, 132 insertions(+), 221 deletions(-) create mode 100644 cluster/services/dns/acme-dns-db-credentials.age create mode 100644 cluster/services/dns/acme-dns-direct-key.age delete mode 100644 cluster/services/dns/admin.nix delete mode 100644 cluster/services/dns/pdns-admin-oidc-secrets.age delete mode 100644 cluster/services/dns/pdns-admin-salt.age delete mode 100644 cluster/services/dns/pdns-admin-secret.age delete mode 100644 cluster/services/dns/pdns-api-key.age delete mode 100644 cluster/services/dns/pdns-db-credentials.age diff --git a/cluster/services/dns/acme-dns-db-credentials.age b/cluster/services/dns/acme-dns-db-credentials.age new file mode 100644 index 0000000..f0b6cb7 --- /dev/null +++ b/cluster/services/dns/acme-dns-db-credentials.age @@ -0,0 +1,16 @@ +age-encryption.org/v1 +-> ssh-ed25519 NO562A YndVtONpmfFXYB1ASnPHsfczl1UbgZ2vccIrX2pEgx0 +VzH2UD583L6wBLMCo6faIGyHR4+zXXOUTgQduEiFOxI +-> ssh-ed25519 5/zT0w +67r5S6PSFEgnrTu3eZpOd3eemZUdDOE+kjUw6GDgUM +jPzlW7hePFgsABUjryePu5yergQ2Qjczmmoxuo6CK+U +-> ssh-ed25519 TCgorQ DGJPjJYpeibxM+8OwofUCdttIT2OdNbvQ66wpWQM8XU +JCNQ3bT21j2ZsxbzA6FieKIui6lsvk1p0nvNOT7YtFo +-> ssh-ed25519 d3WGuA hIl5yluwf1f0DP5ZW1MalGPCj4XFYOu2sofwJSQZ6RE +BSHoe4cdRJlPrkc+taUIaIIUknexlGttzz2d9I3jtmk +-> ssh-ed25519 YIaSKQ EbqXS/XFQHSXCbzDJmg4gGUxP9TX3+vOxWtNQDJ8ih4 +hNaWzoFG2iVef4Gm30LilGXYNsVkhmVt9dOvBo02mbM +-> V]i@xRtJ-grease +NEPxMUZa76GclWOasWptt6QS7frMclp9o+kD4KCLJB7ucFOYK7xxWfAEMkjtadfP +m0bbgbw7Jcs9/lA8VNAG2D5jTBayGgpkBQZ4 +--- ViqZD8mJEKIMCZ5Q+wRQWR2FX/LMEfUwoumUtHlYabQ +KAgZ<*DfV6=G+e`p6[ۑ۠-H1Û[fV.H"Ohj8$;ۑ&5xw/m^7f5ԵyώC6Ui-R=/_R==1'Ҡq޷vcw \ No newline at end of file diff --git a/cluster/services/dns/acme-dns-direct-key.age b/cluster/services/dns/acme-dns-direct-key.age new file mode 100644 index 0000000..568ab21 --- /dev/null +++ b/cluster/services/dns/acme-dns-direct-key.age @@ -0,0 +1,21 @@ +age-encryption.org/v1 +-> ssh-ed25519 NO562A 9n5IirzhNBIPRj9Gir+/yQhFH830sgfezsqY5Ulzz3o +VItDDdgfTFcvSq/QpIqTHnfr1VHqfI6nPz+WWKYQjHw +-> ssh-ed25519 5/zT0w MfBZrd8wJjoProwdPqsS9CZ9aYNTXgrYviFDwuchQVM +8WKPYO+i1ZSkPYDrHVJ5Pclj2hEzqwAtf31Agzei444 +-> ssh-ed25519 TCgorQ 3QYtSx/2eiFp54W60F8FlERfHx+DUfnXXfugiXNPECg +pBx3If3qihD//Aq8hDWCt+U1tiWoCLUDcg/RyVCD0D0 +-> ssh-ed25519 P/nEqQ NImm+vKuL50G2kdD2svmfkwsovmryCSyKyhnZ0duDDo +U0PTKHiCj4SxomnJdgubo+3sStSE+YwvCnrRl7aAS1Q +-> ssh-ed25519 FfIUuQ SRgJoBIoW71SiXuHqlnGqRG5AKUrnQy0ecwznGEGTHA +a0IS3hjMln1tWEjo30A6gYtaV7TJSY4SZDarhahMoLk +-> ssh-ed25519 d3WGuA 0qVNcrYe53Wo46zFJs6UZtX0dq7TUy72WGdGpLqB3yo +jTHE9PfhRw5lbBlfznS+ThkSsab3ioearf91xyPBfdQ +-> ssh-ed25519 YIaSKQ CCcBlAOms2aSkB6pws6tN+4Gf551idI9Zq0rokd0P1c +/3oFp6hf+jggurbcuu0cXdDL8lr6m/LTHEeNgiJt2gg +-> K&wn-grease ,Ewz Jc+dQQRp NU~. +FvDOuTGNaLuCfDelsrRbthjuJT9fBZAQ+kz+7Stoc2wciXV1YpCcOYDHSF38OwRF +X/pyjVudbJKS0Mphda6phw +--- 3JFwCzeJsIgRkTpmy9MAvQ64BCZoa98kNKOuT57WI6Y +& Op-P.+"jG +sgnz[t DRĽmlxp)X&!gUC6`>%%LyvEB$r3i-4k8FHin#Xs1D_iPP>^{=JFntAThEIS-bDdu11!r3 z3Ve(tQ3lj>lEPS!8N(E{V9b`c`&uPWJ+j3nc;M9*=~=#El3Xllj$L?04hZ0t@xoT?xdVi z>6VXloj~OzpKcpd?1}`SuM*vit8GLiSR}_0e9EOZ8xmUs#^k5IvnkZ0hU=Q?yga}8 z_*kl0FzNSDoXcy(by9J`PZJ-8AJPoE3YlcwM5ZJZ^CE`36Ql%2RH|4$O9&`D(%3DQ zlhg&{=m7jNxIQ#YZ(-ZFMZv}mEbn{>> i-v4lI?c&`_+UXDF&)@6s4_}|Y!=6z~d;R3bm7RYA2efnm diff --git a/cluster/services/dns/pdns-admin-salt.age b/cluster/services/dns/pdns-admin-salt.age deleted file mode 100644 index 1e4d774..0000000 --- a/cluster/services/dns/pdns-admin-salt.age +++ /dev/null @@ -1,11 +0,0 @@ -age-encryption.org/v1 --> ssh-ed25519 NO562A d/YNanH/cHoFLPp8WcCXHh/LQLRwaUa95JiRLbgb8RI -UPEHpnHHTU6dGKi2MbApEspcpt1lFtFZ4XJjShL7OoE --> ssh-ed25519 5/zT0w Rv9ZS5P2Eca3npPLR7yym/XTRSDfVmgRwH1pAGR79T8 -4A/KXc2wxxokfDAwWYf0ZTUEzQ8ldkC+zRNZY3KjBTs --> ssh-ed25519 d3WGuA 2R0kaVjuhU3wT9pjj214zkEaHYNSlMxf9Z+MfBssHwY -EU5LWk6xfohWM/3sAqYtUvFmRgIPxOLXHnlqbsQ3+ok --> -|(-grease W=cc~ O2q5 -FZzh/ZwDS2EqvVZ9NErmUwCMN72op1Qy ---- Ducan3ugRJC3dmWLr7+FKok+WmInOgOzW0ccYeqAFAQ -*Q.SCf*`5w"~xw*\"t '0L \ No newline at end of file diff --git a/cluster/services/dns/pdns-admin-secret.age b/cluster/services/dns/pdns-admin-secret.age deleted file mode 100644 index 03f5ab0..0000000 --- a/cluster/services/dns/pdns-admin-secret.age +++ /dev/null @@ -1,12 +0,0 @@ -age-encryption.org/v1 --> ssh-ed25519 NO562A hUR+UdHnpazhANM8DKToI5Th3lv1aAuxZ1IQKvCOv34 -PvsiSym8YdleDULLnWuTs1x08KO3EmAg/AAjulgrgqE --> ssh-ed25519 5/zT0w qMXS2xLOLv/+l6brG11i+3FwHdrhlmxZBNtBiU9hu2g -BlFYPvH4mFJRMHTlHwnBdJb6QcugylwZuT5bgSKcQa0 --> ssh-ed25519 d3WGuA k2fRQ3+HyZP+bb/gkVKQqUmbITJLPm9tGp67DbRfiCs -RX9CACfYpYKvSqyfXjvEokTGsp4+ECQBD8i1ehD5xRg --> IB@F$9G-grease -cXRgUVdIPGEjft1CJA ---- si16Det/GwF7GLHLt0ha8v4rFFeJXyhEylIiqzZVAK8 -ְpǺ#4^ ~u UuaQBj(N)q<"%,V95Zh#W["M&+%SQBޛy#ϫtwq, 3YyIq}ʓ>sgzs ƆFP|=~KQR,DZu+պZGHa=;C.uVSh$VA9= ? \ No newline at end of file diff --git a/cluster/services/dns/pdns-api-key.age b/cluster/services/dns/pdns-api-key.age deleted file mode 100644 index b138c82e6e8e2fe9985385cafd976bfe5c1594af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1279 zcmZY5+pp6E0Dy5W3ZCMFMkAmYXHL+VIofsYy2;IT+v#O%yRK`yRgtV`T`ybL%eu9l z5{$%%F$gANB1Rub#9)N@fJ(#|NQfaS0X0E_5(N?-d@u(kToS>{>+vu6<@?&YmeK~Z ze(VOOGg#yJIyg=PsSSzY&`4<=7)8_h1k0gWxRBu8sGMamGcCY!LBd@`ZwKYT9w^b+ z<@2%^h8;6jXwJuHfia&OP(CS&yqV`YSkKV?icSg|BMCXdAUg^dGPWheb)q$2L%~S{ z3KOQ-tBkB967jIp&{_rpda<4FvUD$qoLQ_eFq1`|2}@|Y1=vglR^be0N%FxhkHcKW+5TlzK(XjF* z88ZTzu8d8s?*KSP5}}rBQlc>K^G$((2vCEgu3UxEW0I(gby<*wX4_Gz`C1Jels!H{ z#b5_zlJy={M#t@HH6yWzEe2X&Z{aHE>NHb>@SdfhRvAFt5Q6Fyib^8h)(~G5+!?d{ z?^d_$;Z1c?Y?|4M!HmMFFkhSLQsRhDv)dCNWCt%3;KMmlaYy0AyI7< zbFSeeGmeZ$9jOloqkgU1o;6*K^hPBpZt$$acWMkz0a#K~R8Zl}C@rZ3nJ@KpY?y@@ z6)b9HIp{fgiFK`r%pw0f*`nG)g-?h{#x0u-oX9XXELEXyN1^ZuVjEW5^-zzF?NZe< z%hehj0!hpnFgaBv=uNJVTP>hJ z@L?d8N+lGj!z-Cw7N&}Y0Ahy|rJ&Tg9xS%_cvKQZB=le~V?b_}0kn^EKt8_cy!m~2 zxOViVEONM4tzDDzXYh^Y?!BeU(w}RucON-$^`5I2t|T9Q;e65lrFCw_g5}(%&Cjja zbKj1eh1Whw2VXsPX8h{1;reeDe8{c4_vY(AZeMcX%%R6mo9}*P?YV39{rs_&Tau?Q z&mG-+{MXHkr{CB->Atw&{Py~e6Ttp`YyVn*>2vDJ^p>gH4;dFIgEnHxf&)-8sDA?tE+E+3DuDJ0IIMy^&gUYyU&m?UmaOeC>R( z5<=2ZBQQX{p{IQ q-~AzMUAXv}Lzk_CKmUEoy%CPQ8O)djB6rtH)jd diff --git a/cluster/services/dns/pdns-db-credentials.age b/cluster/services/dns/pdns-db-credentials.age deleted file mode 100644 index c25e95a..0000000 --- a/cluster/services/dns/pdns-db-credentials.age +++ /dev/null @@ -1,20 +0,0 @@ -age-encryption.org/v1 --> ssh-ed25519 NO562A OQaDWMrfvfQoluWFIldZgZFEdqzFfXhPvO6BqOZofnU -qoUEZlKSTNJ53jgTK9eP2GDJogugtCfKqBaVH7mCqZY --> ssh-ed25519 5/zT0w U5w9w/DE+zDgw4YI6DDVAMSaAAcR+3+BIioVXAGMfHg -9Ps2qB+P2DWDdYPRPuzmBECWzJ90LVq8B71LlrO0Gyk --> ssh-ed25519 TCgorQ s91OjOZH6825aSBRfiSN+ODBOJvbjff6s2fzf/8o2Wk -zJI/5oKwagyOJUy1siwAcZ7wcsEMUyekYjP7TlsAjoY --> ssh-ed25519 d3WGuA 1gPF8W/p+wVclVrMGbvnBAO9IvSX9G8qNEaKpHeX23w -L4N6MxD5SeEhqcjRx1e8M/rMtK2Qg+elYgKCHkHi71o --> ssh-ed25519 YIaSKQ eOwUbPa6RceRM4zsB8lHSCYtSJoLX1Fqs8CdzM7qkCQ -8OPkkFP0B+uN0zBZAUmEgogp97YO+qlvsG6wnMwkzLw --> L_-grease 51PFh7A -k9hZ2FbD3JDWGN8/WFjOCM0Ud/uvQhZZDceL/Esa8cfp ---- v5Noo1KII/WFJxNGjEO2hqdhgHdastilx/M1vFos5dE -mܴRx ;Hp -saᙵ 4y YQHPe 00[ -Ԏy'|2[qېWS/d.Q49,͆}o GOkrGMG& -4"8.m槫7Pku @XA$ >·+| Vtn|C>\a2 -{Us ٠ςbɇg.s3M24[+UD!Pش7[_>a3 - -s \ No newline at end of file diff --git a/secrets.nix b/secrets.nix index 3e1ff3a..415e348 100644 --- a/secrets.nix +++ b/secrets.nix @@ -13,11 +13,8 @@ in with hosts; "cluster/services/cachix-deploy-agent/credentials/prophet.age".publicKeys = max ++ map systemKeys [ prophet ]; "cluster/services/cachix-deploy-agent/credentials/VEGAS.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "cluster/services/cachix-deploy-agent/credentials/thunderskin.age".publicKeys = max ++ map systemKeys [ thunderskin ]; - "cluster/services/dns/pdns-admin-oidc-secrets.age".publicKeys = max ++ map systemKeys [ VEGAS ]; - "cluster/services/dns/pdns-admin-salt.age".publicKeys = max ++ map systemKeys [ VEGAS ]; - "cluster/services/dns/pdns-admin-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ]; - "cluster/services/dns/pdns-api-key.age".publicKeys = max ++ map systemKeys [ checkmate grail thunderskin VEGAS prophet ]; - "cluster/services/dns/pdns-db-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ]; + "cluster/services/dns/acme-dns-direct-key.age".publicKeys = max ++ map systemKeys [ checkmate grail thunderskin VEGAS prophet ]; + "cluster/services/dns/acme-dns-db-credentials.age".publicKeys = max ++ map systemKeys [ checkmate VEGAS prophet ]; "cluster/services/forge/credentials/forgejo-oidc-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "cluster/services/forge/credentials/forgejo-db-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS ]; "cluster/services/hercules-ci-multi-agent/secrets/hci-cache-config.age".publicKeys = max ++ map systemKeys [ VEGAS prophet ]; From a09b8ff7c577d7307378f5b42f1d3baba9eb5e69 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 16:32:00 +0100 Subject: [PATCH 21/47] cluster/services/dns: create dns records for machines --- cluster/services/dns/default.nix | 1 + cluster/services/dns/nodes.nix | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 cluster/services/dns/nodes.nix diff --git a/cluster/services/dns/default.nix b/cluster/services/dns/default.nix index f6b3419..5f3226f 100644 --- a/cluster/services/dns/default.nix +++ b/cluster/services/dns/default.nix @@ -7,6 +7,7 @@ in { imports = [ ./options.nix + ./nodes.nix ]; links = { diff --git a/cluster/services/dns/nodes.nix b/cluster/services/dns/nodes.nix new file mode 100644 index 0000000..c825837 --- /dev/null +++ b/cluster/services/dns/nodes.nix @@ -0,0 +1,11 @@ +{ depot, lib, ... }: + +{ + dns.records = lib.mapAttrs' (name: hour: { + name = lib.toLower "${name}.${hour.enterprise.subdomain}"; + value = { + type = "A"; + target = [ hour.interfaces.primary.addrPublic ]; + }; + }) depot.gods.fromLight; +} From afb95e1d3bf1e745ee73f7d7d211a55748484cd1 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 16:32:36 +0100 Subject: [PATCH 22/47] cluster/services/mail: init --- cluster/services/mail/default.nix | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 cluster/services/mail/default.nix diff --git a/cluster/services/mail/default.nix b/cluster/services/mail/default.nix new file mode 100644 index 0000000..f271d54 --- /dev/null +++ b/cluster/services/mail/default.nix @@ -0,0 +1,43 @@ +{ depot, ... }: + +{ + dns.records = let + inherit (depot.lib.meta) domain adminEmail; + mailServerAddr = depot.hours.VEGAS.interfaces.primary.addrPublic; + mxAlias = { + type = "CNAME"; + target = [ "mx.${domain}." ]; + }; + in { + mx = { + type = "A"; + target = [ mailServerAddr ]; + }; + smtp = mxAlias; + imap = mxAlias; + mail = mxAlias; + MX = { + name = "@"; + type = "MX"; + target = [ "0 mx.${domain}." ]; + }; + # compat for old email aliases + "max.admin" = { + type = "MX"; + target = [ "0 mx.${domain}." ]; + }; + SPF = { + name = "@"; + type = "TXT"; + target = [ "v=spf1 mx a ip4:${mailServerAddr} ~all" ]; + }; + _dmarc = { + type = "TXT"; + target = [ "v=DMARC1; p=reject; rua=mailto:${adminEmail}; ruf=mailto:${adminEmail}; sp=quarantine; ri=604800" ]; + }; + "${domain}._domainkey" = { + type = "TXT"; + target = [ "v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC9Q5VrGWEcG/CWZSWJl0tRQR3uiOkPH7AcNH+H7Gpa5S/E7tLZNyWuKOmNCRi/FKeqXcD5zIfI1sYsWZKOE70Un/ShCdRUzwD1Em8bO6yz/BbY1cBxHBQdCrH2ylMgn3UW0X1rM75EgJntAYkOqovtL78BtDbUhagO/0MTFpySpQIDAQAB" ]; + }; + }; +} From eae6934b92f50cd0360460b32d829bcaa8d0a73d Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 17:29:27 +0100 Subject: [PATCH 23/47] cluster/services/dns: add nameserver records --- cluster/services/dns/default.nix | 1 + cluster/services/dns/ns-records.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 cluster/services/dns/ns-records.nix diff --git a/cluster/services/dns/default.nix b/cluster/services/dns/default.nix index 5f3226f..6c2ed43 100644 --- a/cluster/services/dns/default.nix +++ b/cluster/services/dns/default.nix @@ -8,6 +8,7 @@ in imports = [ ./options.nix ./nodes.nix + ./ns-records.nix ]; links = { diff --git a/cluster/services/dns/ns-records.nix b/cluster/services/dns/ns-records.nix new file mode 100644 index 0000000..7170ec0 --- /dev/null +++ b/cluster/services/dns/ns-records.nix @@ -0,0 +1,26 @@ +{ config, depot, lib, ... }: + +let + cfg = config.services.dns; + + nsNodes = lib.imap1 (idx: node: { + name = "eu${toString idx}.ns"; + value = { + type = "A"; + target = [ depot.hours.${node}.interfaces.primary.addrPublic ]; + }; + }) cfg.nodes.authoritative; +in + +{ + dns.records = lib.mkMerge [ + (lib.listToAttrs nsNodes) + { + NS = { + name = "@"; + type = "NS"; + target = map (ns: "${ns.name}.${depot.lib.meta.domain}.") nsNodes; + }; + } + ]; +} From 93ceb5c0ea7fac60d083022081c44d60cee56acb Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 17:51:48 +0100 Subject: [PATCH 24/47] cluster/services/websites: add top-level dns record --- cluster/services/websites/default.nix | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cluster/services/websites/default.nix b/cluster/services/websites/default.nix index 3aec785..8ec73b5 100644 --- a/cluster/services/websites/default.nix +++ b/cluster/services/websites/default.nix @@ -51,7 +51,16 @@ in }; }; - dns.records = lib.genAttrs [ "www" "draw" "stop-using-nix-env" "whoami" ] (lib.const { - consulService = "static-lb"; - }); + dns.records = lib.mkMerge [ + (lib.genAttrs [ "www" "draw" "stop-using-nix-env" "whoami" ] (lib.const { + consulService = "static-lb"; + })) + { + CNAME = { + name = "@"; + type = "CNAME"; + target = [ "www.${domain}." ]; + }; + } + ]; } From b24f73bc4b6cdab45ecda48a8ca0c3af4ed72481 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:04:15 +0100 Subject: [PATCH 25/47] cluster/services/idm: add dns records --- cluster/services/idm/default.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cluster/services/idm/default.nix b/cluster/services/idm/default.nix index c8a5b80..279316d 100644 --- a/cluster/services/idm/default.nix +++ b/cluster/services/idm/default.nix @@ -34,4 +34,22 @@ ]; }; }; + + dns.records = let + serverAddrsPublic = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.idm.nodes.server; + serverAddrsInternal = map + (node: config.vars.mesh.${node}.meshIp) + config.services.idm.nodes.server; + in { + idm = { + type = "A"; + target = serverAddrsPublic; + }; + "idm-ldap.internal" = { + type = "A"; + target = serverAddrsInternal; + }; + }; } From 7d7714db4c3aa1898120695472802c8893108aa9 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:09:24 +0100 Subject: [PATCH 26/47] cluster/services/search: add dns records --- cluster/services/search/default.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cluster/services/search/default.nix b/cluster/services/search/default.nix index ab0b4c2..63efa8e 100644 --- a/cluster/services/search/default.nix +++ b/cluster/services/search/default.nix @@ -1,4 +1,4 @@ -{ depot, ... }: +{ config, depot, ... }: { services.search = { @@ -10,4 +10,8 @@ address = "https://search.${depot.lib.meta.domain}/healthz"; module = "https2xx"; }; + + dns.records.search.target = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.search.nodes.host; } From 4aadf0c482c8baae920d12e4ca66e01732deae65 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:11:50 +0100 Subject: [PATCH 27/47] cluster/services/forge: add dns records --- cluster/services/forge/default.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster/services/forge/default.nix b/cluster/services/forge/default.nix index 7607d5b..2c51555 100644 --- a/cluster/services/forge/default.nix +++ b/cluster/services/forge/default.nix @@ -1,6 +1,12 @@ +{ config, depot, ... }: + { services.forge = { nodes.server = [ "VEGAS" ]; nixos.server = ./server.nix; }; + + dns.records.forge.target = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.forge.nodes.server; } From 38d22c1964a5464821063184faad918d7023e33c Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:13:08 +0100 Subject: [PATCH 28/47] cluster/services/warehouse: add dns records --- cluster/services/warehouse/default.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster/services/warehouse/default.nix b/cluster/services/warehouse/default.nix index e710faa..78f53e3 100644 --- a/cluster/services/warehouse/default.nix +++ b/cluster/services/warehouse/default.nix @@ -1,6 +1,12 @@ +{ config, depot, ... }: + { services.warehouse = { nodes.host = [ "VEGAS" ]; nixos.host = [ ./host.nix ]; }; + + dns.records.warehouse.target = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.warehouse.nodes.host; } From cb8744b99ae5225a28479436027602fb453562d6 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:17:12 +0100 Subject: [PATCH 29/47] cluster/services/matrix: add dns records --- cluster/services/matrix/default.nix | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cluster/services/matrix/default.nix b/cluster/services/matrix/default.nix index cf6e9d2..5c4a962 100644 --- a/cluster/services/matrix/default.nix +++ b/cluster/services/matrix/default.nix @@ -1,4 +1,4 @@ -{ depot, ... }: +{ config, depot, ... }: { services.matrix = { @@ -16,4 +16,15 @@ address = "https://matrix.${depot.lib.meta.domain}/_matrix/federation/v1/version"; module = "https2xx"; }; + + dns.records = let + homeserverAddrs = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.matrix.nodes.homeserver; + in { + matrix.target = homeserverAddrs; + chat.target = homeserverAddrs; + stun.target = homeserverAddrs; + turn.target = homeserverAddrs; + }; } From bbaf0b0c14f4ebc5e392b29fe1f7ab0580d0dcf0 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:18:38 +0100 Subject: [PATCH 30/47] cluster/services/soda: add dns records --- cluster/services/soda/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/services/soda/default.nix b/cluster/services/soda/default.nix index dd74dfe..708f2f3 100644 --- a/cluster/services/soda/default.nix +++ b/cluster/services/soda/default.nix @@ -5,4 +5,6 @@ address = "soda.int.${depot.lib.meta.domain}:22"; module = "sshConnect"; }; + + dns.records.soda.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; } From 38d8894676600e060aec04f694d7b4aa93908a97 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:22:38 +0100 Subject: [PATCH 31/47] cluster/services/nextcloud: add dns records --- cluster/services/nextcloud/default.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cluster/services/nextcloud/default.nix b/cluster/services/nextcloud/default.nix index f349432..6a1e1a8 100644 --- a/cluster/services/nextcloud/default.nix +++ b/cluster/services/nextcloud/default.nix @@ -1,4 +1,4 @@ -{ depot, ... }: +{ config, depot, ... }: { services.nextcloud = { @@ -10,4 +10,8 @@ address = "https://storage.${depot.lib.meta.domain}/status.php"; module = "nextcloudStatus"; }; + + dns.records.storage.target = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.nextcloud.nodes.host; } From e961260700270ecd89f9582a512351ea3d8c664b Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:28:51 +0100 Subject: [PATCH 32/47] cluster/services/object-storage: add dns records --- cluster/services/object-storage/default.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cluster/services/object-storage/default.nix b/cluster/services/object-storage/default.nix index bc7d54e..4835a27 100644 --- a/cluster/services/object-storage/default.nix +++ b/cluster/services/object-storage/default.nix @@ -1,4 +1,4 @@ -{ depot, ... }: +{ config, depot, ... }: { services.object-storage = { @@ -10,4 +10,14 @@ address = "https://object-storage.${depot.lib.meta.domain}/minio/health/live"; module = "https2xx"; }; + + dns.records = let + serverAddrs = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.object-storage.nodes.host; + in { + object-storage.target = serverAddrs; + "console.object-storage".target = serverAddrs; + cdn.target = serverAddrs; + }; } From 001f6cd078557c4c709b2255c58c1e3eaa9aa65e Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:34:07 +0100 Subject: [PATCH 33/47] cluster/services/fbi: init --- cluster/services/fbi/default.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cluster/services/fbi/default.nix diff --git a/cluster/services/fbi/default.nix b/cluster/services/fbi/default.nix new file mode 100644 index 0000000..5a5805c --- /dev/null +++ b/cluster/services/fbi/default.nix @@ -0,0 +1,12 @@ +{ depot, ... }: + +{ + dns.records = let + fbiAddr = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; + in { + fbi-index.target = fbiAddr; + fbi-requests.target = fbiAddr; + radarr.target = fbiAddr; + sonarr.target = fbiAddr; + }; +} From 6d22f7bdb7f7e8480c77d7dbb05a30cacf14fe63 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:34:56 +0100 Subject: [PATCH 34/47] cluster/services/meet: add dns records --- cluster/services/meet/default.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster/services/meet/default.nix b/cluster/services/meet/default.nix index 9322b22..118fa19 100644 --- a/cluster/services/meet/default.nix +++ b/cluster/services/meet/default.nix @@ -1,6 +1,12 @@ +{ config, depot, ... }: + { services.meet = { nodes.host = [ "prophet" ]; nixos.host = ./host.nix; }; + + dns.records.meet.target = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.meet.nodes.host; } From 9abd4b6c0a867afa4dd6ea4ee6e75141227a395c Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:39:58 +0100 Subject: [PATCH 35/47] cluster/services/attic: add dns records --- cluster/services/attic/default.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cluster/services/attic/default.nix b/cluster/services/attic/default.nix index 83dd422..f358ee1 100644 --- a/cluster/services/attic/default.nix +++ b/cluster/services/attic/default.nix @@ -1,3 +1,5 @@ +{ config, depot, ... }: + { services.attic = { nodes = { @@ -18,4 +20,13 @@ allow.attic = [ "read" "write" ]; }; }; + + dns.records = let + serverAddrs = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.attic.nodes.server; + in { + cache-api.target = serverAddrs; + cache.target = serverAddrs; + }; } From 0a6755dac5e1d6b91c4a127c8f5897dbbad3c454 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:53:10 +0100 Subject: [PATCH 36/47] cluster/services/sso: init --- cluster/services/sso/default.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cluster/services/sso/default.nix diff --git a/cluster/services/sso/default.nix b/cluster/services/sso/default.nix new file mode 100644 index 0000000..45c2292 --- /dev/null +++ b/cluster/services/sso/default.nix @@ -0,0 +1,10 @@ +{ depot, ... }: + +{ + dns.records = let + ssoAddr = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; + in { + login.target = ssoAddr; + account.target = ssoAddr; + }; +} From bde04dac875fef9d2574d1a3f90c2ca58618c0d6 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 3 Dec 2023 23:59:13 +0100 Subject: [PATCH 37/47] cluster/services/websites: add dns records for old sites --- cluster/services/websites/default.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cluster/services/websites/default.nix b/cluster/services/websites/default.nix index 8ec73b5..999151b 100644 --- a/cluster/services/websites/default.nix +++ b/cluster/services/websites/default.nix @@ -51,7 +51,9 @@ in }; }; - dns.records = lib.mkMerge [ + dns.records = let + oldStaticAddr = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; + in lib.mkMerge [ (lib.genAttrs [ "www" "draw" "stop-using-nix-env" "whoami" ] (lib.const { consulService = "static-lb"; })) @@ -61,6 +63,16 @@ in type = "CNAME"; target = [ "www.${domain}." ]; }; + + autoconfig.target = oldStaticAddr; + + ktp.target = oldStaticAddr; + legacy.target = oldStaticAddr; + + # jokes + "bone-ds-dc.com-ldap".target = oldStaticAddr; + rzentrale.target = oldStaticAddr; + wunschnachricht.target = oldStaticAddr; } ]; } From 5150894720b35474103e3d26faf18414009cb5b9 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 00:01:33 +0100 Subject: [PATCH 38/47] cluster/services/ipfs: more dns records --- cluster/services/ipfs/default.nix | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cluster/services/ipfs/default.nix b/cluster/services/ipfs/default.nix index 71d2f2f..77e48cc 100644 --- a/cluster/services/ipfs/default.nix +++ b/cluster/services/ipfs/default.nix @@ -52,11 +52,15 @@ dns.records = { p2p.consulService = "ipfs-gateway"; - "\\.ipfs" = { + pin.consulService = "ipfs-gateway"; + "ipfs.admin".target = map + (node: depot.hours.${node}.interfaces.primary.addrPublic) + config.services.ipfs.nodes.remote-api; + "^[^_].+\\.ipfs" = { consulService = "ipfs-gateway"; rewrite.type = "regex"; }; - "\\.ipns" = { + "^[^_].+\\.ipns" = { consulService = "ipfs-gateway"; rewrite.type = "regex"; }; From 195fe56279e9dfed2cbacd3ce0817aaf72a4adca Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 00:06:07 +0100 Subject: [PATCH 39/47] cluster/services/bitwarden: init --- cluster/services/bitwarden/default.nix | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 cluster/services/bitwarden/default.nix diff --git a/cluster/services/bitwarden/default.nix b/cluster/services/bitwarden/default.nix new file mode 100644 index 0000000..5f8e676 --- /dev/null +++ b/cluster/services/bitwarden/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +{ + dns.records.keychain.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; +} From f1e68e7e28b35bb55db458a63c884f328e656077 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 00:08:59 +0100 Subject: [PATCH 40/47] cluster/services/reflex: init --- cluster/services/reflex/default.nix | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 cluster/services/reflex/default.nix diff --git a/cluster/services/reflex/default.nix b/cluster/services/reflex/default.nix new file mode 100644 index 0000000..ccb6685 --- /dev/null +++ b/cluster/services/reflex/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +{ + dns.records.reflex.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; +} From 2a49d440f75f80412f30360078792889ac901e99 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 00:09:37 +0100 Subject: [PATCH 41/47] cluster/services/vault: init --- cluster/services/vault/default.nix | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 cluster/services/vault/default.nix diff --git a/cluster/services/vault/default.nix b/cluster/services/vault/default.nix new file mode 100644 index 0000000..bc17725 --- /dev/null +++ b/cluster/services/vault/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +{ + dns.records.vault.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; +} From 779429c28908ca22a7b3ce00f3bf7d88150c1d71 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 00:09:53 +0100 Subject: [PATCH 42/47] cluster/services/gitlab: init --- cluster/services/gitlab/default.nix | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 cluster/services/gitlab/default.nix diff --git a/cluster/services/gitlab/default.nix b/cluster/services/gitlab/default.nix new file mode 100644 index 0000000..a0c4eef --- /dev/null +++ b/cluster/services/gitlab/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +{ + dns.records.git.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; +} From 0fef0fca669ba7b7debb4db5d361f1422a7a2bec Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 00:10:31 +0100 Subject: [PATCH 43/47] cluster/services/n8n: init --- cluster/services/n8n/default.nix | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 cluster/services/n8n/default.nix diff --git a/cluster/services/n8n/default.nix b/cluster/services/n8n/default.nix new file mode 100644 index 0000000..85d5a73 --- /dev/null +++ b/cluster/services/n8n/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +{ + dns.records.api.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; +} From 3231b65a26310f749ddca19a5d500a06ce4c81a5 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 00:25:06 +0100 Subject: [PATCH 44/47] cluster/services/cdn-shield: init --- cluster/services/cdn-shield/default.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cluster/services/cdn-shield/default.nix diff --git a/cluster/services/cdn-shield/default.nix b/cluster/services/cdn-shield/default.nix new file mode 100644 index 0000000..b324b1e --- /dev/null +++ b/cluster/services/cdn-shield/default.nix @@ -0,0 +1,12 @@ +{ depot, ... }: + +{ + dns.records = let + cdnShieldAddr = [ depot.hours.VEGAS.interfaces.primary.addrPublic ]; + in { + "fonts-googleapis-com.cdn-shield".target = cdnShieldAddr; + "fonts-gstatic-com.cdn-shield".target = cdnShieldAddr; + "cdnjs-cloudflare-com.cdn-shield".target = cdnShieldAddr; + "wttr-in.cdn-shield".target = cdnShieldAddr; + }; +} From bfd7a4214c7ba6881d80845115bc9f3953524d3f Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 19:23:31 +0100 Subject: [PATCH 45/47] cluster/services/acme-client: switch to acme-dns with custom script --- cluster/services/acme-client/client.nix | 47 +++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/cluster/services/acme-client/client.nix b/cluster/services/acme-client/client.nix index 53cae09..ecc92c1 100644 --- a/cluster/services/acme-client/client.nix +++ b/cluster/services/acme-client/client.nix @@ -1,10 +1,45 @@ -{ cluster, config, pkgs, ... }: +{ cluster, config, depot, lib, pkgs, ... }: + +let + authoritativeServers = map + (node: cluster.config.hostLinks.${node}.dnsAuthoritative.tuple) + cluster.config.services.dns.nodes.authoritative; + + execScript = pkgs.writeShellScript "acme-dns-exec" '' + action="$1" + subdomain="''${2%.${depot.lib.meta.domain}.}" + key="$3" + umask 77 + source "$EXEC_ENV_FILE" + headersFile="$(mktemp)" + echo "X-Direct-Key: $ACME_DNS_DIRECT_STATIC_KEY" > "$headersFile" + case "$action" in + present) + for i in {1..5}; do + ${pkgs.curl}/bin/curl -X POST -s -f -H "@$headersFile" \ + "${cluster.config.links.acmeDnsApi.url}/update" \ + --data '{"subdomain":"'"$subdomain"'","txt":"'"$key"'"}' && break + sleep 5 + done + ;; + esac + ''; +in { - age.secrets.pdns-api-key-acme = cluster.config.vars.pdns-api-key-secret // { owner = "acme"; }; + age.secrets.acmeDnsApiKey = { + file = ../dns/acme-dns-direct-key.age; + owner = "acme"; + }; - security.acme.defaults.credentialsFile = pkgs.writeText "acme-pdns-credentials" '' - PDNS_API_URL=${cluster.config.links.powerdns-api.url} - PDNS_API_KEY_FILE=${config.age.secrets.pdns-api-key-acme.path} - ''; + security.acme.defaults = { + extraLegoFlags = lib.flatten [ + (map (x: [ "--dns.resolvers" x ]) authoritativeServers) + "--dns-timeout" "30" + ]; + credentialsFile = pkgs.writeText "acme-exec-config" '' + EXEC_PATH=${execScript} + EXEC_ENV_FILE=${config.age.secrets.acmeDnsApiKey.path} + ''; + }; } From 0943c410c3637306493f283adcc8583c87f2e20b Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 19:31:03 +0100 Subject: [PATCH 46/47] cluster: switch to exec dns01 provider --- cluster/services/certificates/internal-wildcard.nix | 2 +- cluster/services/dns/coredns.nix | 2 +- cluster/services/idm/server.nix | 2 +- cluster/services/ipfs/cluster.nix | 2 +- cluster/services/ipfs/gateway.nix | 4 ++-- cluster/services/irc/irc-host.nix | 2 +- cluster/services/monitoring/grafana-ha.nix | 2 +- cluster/services/storage/garage-gateway.nix | 2 +- cluster/services/websites/default.nix | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cluster/services/certificates/internal-wildcard.nix b/cluster/services/certificates/internal-wildcard.nix index 596bfff..8e1c3ca 100644 --- a/cluster/services/certificates/internal-wildcard.nix +++ b/cluster/services/certificates/internal-wildcard.nix @@ -11,7 +11,7 @@ in security.acme.certs."internal.${domain}" = { domain = "*.internal.${domain}"; extraDomainNames = [ "*.internal.${domain}" ]; - dnsProvider = "pdns"; + dnsProvider = "exec"; group = "nginx"; postRun = '' ${pkgs.acl}/bin/setfacl -Rb out/ diff --git a/cluster/services/dns/coredns.nix b/cluster/services/dns/coredns.nix index 85b3c21..e4bbf35 100644 --- a/cluster/services/dns/coredns.nix +++ b/cluster/services/dns/coredns.nix @@ -42,7 +42,7 @@ in }; security.acme.certs."securedns.${domain}" = { - dnsProvider = "pdns"; + dnsProvider = "exec"; # using a different ACME provider because Android Private DNS is fucky server = "https://api.buypass.com/acme/directory"; reloadServices = [ diff --git a/cluster/services/idm/server.nix b/cluster/services/idm/server.nix index 9630e0a..af9b91a 100644 --- a/cluster/services/idm/server.nix +++ b/cluster/services/idm/server.nix @@ -18,7 +18,7 @@ in security.acme.certs = { "internal.${domain}".reloadServices = [ "kanidm.service" ]; "idm.${domain}" = { - dnsProvider = "pdns"; + dnsProvider = "exec"; webroot = lib.mkForce null; }; }; diff --git a/cluster/services/ipfs/cluster.nix b/cluster/services/ipfs/cluster.nix index 7ed288d..14d90b6 100644 --- a/cluster/services/ipfs/cluster.nix +++ b/cluster/services/ipfs/cluster.nix @@ -81,7 +81,7 @@ in { services.nginx.virtualHosts."pin.${domain}" = vhosts.proxy "http://unix:${pinSvcSocket}"; users.users.nginx.extraGroups = [ cfg.group ]; security.acme.certs."pin.${domain}" = { - dnsProvider = "pdns"; + dnsProvider = "exec"; webroot = lib.mkForce null; }; } diff --git a/cluster/services/ipfs/gateway.nix b/cluster/services/ipfs/gateway.nix index ef0e97f..b5bbe4d 100644 --- a/cluster/services/ipfs/gateway.nix +++ b/cluster/services/ipfs/gateway.nix @@ -48,12 +48,12 @@ in security.acme.certs."ipfs.${domain}" = { domain = "*.ipfs.${domain}"; extraDomainNames = [ "*.ipns.${domain}" ]; - dnsProvider = "pdns"; + dnsProvider = "exec"; group = "nginx"; }; security.acme.certs."p2p.${domain}" = { - dnsProvider = "pdns"; + dnsProvider = "exec"; webroot = lib.mkForce null; }; diff --git a/cluster/services/irc/irc-host.nix b/cluster/services/irc/irc-host.nix index 42e1c47..211043e 100644 --- a/cluster/services/irc/irc-host.nix +++ b/cluster/services/irc/irc-host.nix @@ -82,7 +82,7 @@ in { params.ngircd.bits = 2048; }; security.acme.certs."${serverName}" = { - dnsProvider = "pdns"; + dnsProvider = "exec"; group = "ngircd"; reloadServices = [ "ngircd" ]; extraDomainNames = [ linkGlobalSecure.ipv4 ]; diff --git a/cluster/services/monitoring/grafana-ha.nix b/cluster/services/monitoring/grafana-ha.nix index af10b4e..b36d5bb 100644 --- a/cluster/services/monitoring/grafana-ha.nix +++ b/cluster/services/monitoring/grafana-ha.nix @@ -103,7 +103,7 @@ in }; security.acme.certs."monitoring.${domain}" = { - dnsProvider = "pdns"; + dnsProvider = "exec"; webroot = lib.mkForce null; }; diff --git a/cluster/services/storage/garage-gateway.nix b/cluster/services/storage/garage-gateway.nix index 5ba70b9..3a83738 100644 --- a/cluster/services/storage/garage-gateway.nix +++ b/cluster/services/storage/garage-gateway.nix @@ -20,7 +20,7 @@ in }; }; security.acme.certs.${link.hostname} = { - dnsProvider = "pdns"; + dnsProvider = "exec"; webroot = lib.mkForce null; }; diff --git a/cluster/services/websites/default.nix b/cluster/services/websites/default.nix index 999151b..501ec9c 100644 --- a/cluster/services/websites/default.nix +++ b/cluster/services/websites/default.nix @@ -6,7 +6,7 @@ let acmeUseDNS = name: conf: { name = conf.useACMEHost or conf.serverName or name; value = { - dnsProvider = "pdns"; + dnsProvider = "exec"; webroot = null; }; }; From b0e81bf75a49e93d120311ab1dc969f0171a6222 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 4 Dec 2023 22:59:17 +0100 Subject: [PATCH 47/47] packages/powerdns-admin: drop --- modules/autopatch/default.nix | 1 - packages/patched-derivations.nix | 8 -------- packages/system-filter.nix | 1 - patches/base/powerdns-admin/fix-userinfo.patch | 13 ------------- 4 files changed, 23 deletions(-) delete mode 100644 patches/base/powerdns-admin/fix-userinfo.patch diff --git a/modules/autopatch/default.nix b/modules/autopatch/default.nix index d1e2c90..68da84b 100644 --- a/modules/autopatch/default.nix +++ b/modules/autopatch/default.nix @@ -7,7 +7,6 @@ inherit (patched) kanidm - powerdns-admin prometheus-jitsi-exporter tempo ; diff --git a/packages/patched-derivations.nix b/packages/patched-derivations.nix index cb0998b..5dbe850 100644 --- a/packages/patched-derivations.nix +++ b/packages/patched-derivations.nix @@ -44,14 +44,6 @@ super: rec { postgresql = super.postgresql_14; - powerdns-admin = let - package = super.powerdns-admin.override { - python3 = super.python3.override { - packageOverrides = _: _: { python3-saml = null; }; - }; - }; - in patch package "patches/base/powerdns-admin"; - prometheus-jitsi-exporter = patch super.prometheus-jitsi-exporter "patches/base/prometheus-jitsi-exporter"; s3ql = (patch super.s3ql "patches/base/s3ql").overrideAttrs (old: { diff --git a/packages/system-filter.nix b/packages/system-filter.nix index 42659f1..d0d8a32 100644 --- a/packages/system-filter.nix +++ b/packages/system-filter.nix @@ -5,7 +5,6 @@ hydra = [ "x86_64-linux" ]; jellyfin = [ "x86_64-linux" ]; keycloak = [ "x86_64-linux" ]; - powerdns-admin = [ "x86_64-linux" ]; prometheus-jitsi-exporter = [ "aarch64-linux" ]; searxng = [ "x86_64-linux" ]; tempo = [ "x86_64-linux" ]; diff --git a/patches/base/powerdns-admin/fix-userinfo.patch b/patches/base/powerdns-admin/fix-userinfo.patch deleted file mode 100644 index 93785ea..0000000 --- a/patches/base/powerdns-admin/fix-userinfo.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/powerdnsadmin/routes/index.py b/powerdnsadmin/routes/index.py -index 3a6f55c..417e05f 100644 ---- a/powerdnsadmin/routes/index.py -+++ b/powerdnsadmin/routes/index.py -@@ -392,7 +392,7 @@ def login(): - return authenticate_user(user, 'Azure OAuth') - - if 'oidc_token' in session: -- user_data = json.loads(oidc.get('userinfo').text) -+ user_data = oidc.userinfo() - oidc_username = user_data[Setting().get('oidc_oauth_username')] - oidc_first_name = user_data[Setting().get('oidc_oauth_firstname')] - oidc_last_name = user_data[Setting().get('oidc_oauth_last_name')]