Compare commits

..

2 commits

Author SHA1 Message Date
Max
10d01a0168 flake.lock: Update
Flake lock file updates:

• Updated input 'hercules-ci-agent':
    'github:hercules-ci/hercules-ci-agent/2e10fb21fc2e07edf40763b73443e5934bd40947?narHash=sha256-QDbU8LZzcUSqBp1CBqDj/f5Wd/sdgQ8pZwRWueoMUL4%3D' (2024-07-05)
  → 'github:nrabulinski/hercules-ci-agent/c41b58bf33c9167670685eb97ede9ac240b321d4?narHash=sha256-Dx4kNqDE3//22xyICJ6czV465MUvspowZJlLGSbSGpo%3D' (2024-08-15)
• Removed input 'hercules-ci-agent/haskell-flake'
2024-08-17 00:41:09 +02:00
Max
cc91621e45 meta: switch to IFD-less hercules-ci-agent 2024-08-17 00:41:09 +02:00
54 changed files with 947 additions and 820 deletions

View file

@ -3,14 +3,12 @@
{
hostLinks = lib.pipe config.services [
(lib.filterAttrs (_: svc: svc.meshLinks != {}))
(lib.mapAttrsToList (svcName: svc:
lib.mapAttrsToList (groupName: links:
lib.genAttrs svc.nodes.${groupName} (hostName: lib.mapAttrs (_: cfg: { ... }: {
imports = [ cfg.link ];
ipv4 = config.vars.mesh.${hostName}.meshIp;
}) links)
) svc.meshLinks
))
(lib.mapAttrsToList (svcName: svc: lib.mapAttrsToList (name: cfg: lib.genAttrs svc.nodes.${name} (hostName: {
${cfg.name} = { ... }: {
imports = [ cfg.link ];
ipv4 = config.vars.mesh.${hostName}.meshIp;
};
})) svc.meshLinks))
(map lib.mkMerge)
lib.mkMerge
];

View file

@ -38,8 +38,12 @@ in
};
meshLinks = mkOption {
description = "Create host links on the mesh network.";
type = types.attrsOf (types.attrsOf (types.submodule {
type = types.attrsOf (types.submodule ({ name, ... }: {
options = {
name = mkOption {
type = types.str;
default = "${serviceName}-${name}";
};
link = mkOption {
type = types.deferredModule;
default = {};

View file

@ -0,0 +1,13 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A YQQrnpQI/qyEZugiRwsrPbW4oMYK/rlmRKAdD3JjYz4
JRGFqNc4BVflfR4WUuEOym39IhZlUI778NtOFtxE8eY
-> ssh-ed25519 5/zT0w utH25Xa9WQK9hXbKWsEWK5LJtCbhjpDX6JaomxnRaCI
2MfxxDjs0doUTVsGP9942rx1tyCYsDxhlDo1542BhKQ
-> ssh-ed25519 d3WGuA 6qD02cluQEBqEvupHf93Onlpv8QJJSl/bJm/XqyD+gQ
bLz/ULSaIW6HnPXDKD5dxCbQWv0VC2R+E5wlj7VxOc0
-> Ovax-grease ^1$]}H G4 FpDF XKHkj{
IVdVFYcVe9PoHCCqM3GG1pM6xgTZ5r8XWlkBjlQimgaDArotF4dPpsSTpyc
--- wdTYr6EpFPFsDJI0qQf74c6ce+v5ek6j+mgAx2CI9uI
ÜA³×oÈð:±­‹`ÜVd±å(Kät:fk¼}3*#MJš<4A>Áõ]ê,¤éÐÈÍ69 il`ÛÆJKwAè8­y@Ýœ¯à+&ðÖ©s]ÅÓ–›Ç>~Ší„+Úô
üÁ»<C381>qa©h<C2A9>( YÕ<17>eÇjýI•ê·/ð^å~Ýw Ê
ÆÜßÌZî!^þRˆéÿv­¾…ïkÊp»ÛPÌ)ý̆ÍpÓV5²F΄ÆÚÙÚÞhBÇ»ß b# Š<>´ùºãi”»¸9ìQy¹¾<C2B9>Êè}€ß ƒ¬E}~ZHûjmyq{òxŠÉôß<C3B4>"”éÀ´C#šójÿÐ.ò§y Ô£¸<0A>ÉÐòê<1“Œúâ¾ìßzâš#/êGñ?që

View file

@ -74,7 +74,7 @@ in
serviceConfig = {
Restart = "on-failure";
RestartMaxDelaySec = 30;
RestartSteps = 5;
RestartStesp = 5;
RestartMode = "direct";
};
};

View file

@ -16,7 +16,10 @@
./nar-serve.nix
];
};
meshLinks.server.attic.link.protocol = "http";
meshLinks.server = {
name = "attic";
link.protocol = "http";
};
secrets = let
inherit (config.services.attic) nodes;
in {

View file

@ -9,13 +9,17 @@ let
in
{
imports = [
depot.inputs.attic.nixosModules.atticd
];
services.locksmith.waitForSecrets.atticd = [ "garage-attic" ];
services.atticd = {
enable = true;
package = depot.inputs.attic.packages.attic-server;
environmentFile = secrets.serverToken.path;
credentialsFile = secrets.serverToken.path;
mode = if isMonolith then "monolithic" else "api-server";
settings = {
@ -65,7 +69,6 @@ in
serviceConfig = {
DynamicUser = lib.mkForce false;
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
SystemCallFilter = lib.mkAfter [ "@resources" ];
};
environment = {
AWS_SHARED_CREDENTIALS_FILE = "/run/locksmith/garage-attic";
@ -77,7 +80,6 @@ in
mode = if isMonolith then "manual" else "direct";
definition = {
name = "atticd";
id = "atticd-${config.services.atticd.mode}";
address = link.ipv4;
inherit (link) port;
checks = [

View file

@ -1,36 +0,0 @@
{ lib, ... }:
{
ways.registry.static = { depot, pkgs, ... }: pkgs.writeTextDir "flake-registry.json" (let
flakes = {
depot = {
type = "tarball";
url = "https://forge.${depot.lib.meta.domain}/${depot.lib.meta.domain}/depot/archive/master.tar.gz";
};
depot-nixpkgs = {
type = "github";
owner = "NixOS";
repo = "nixpkgs";
inherit (depot.inputs.nixpkgs.sourceInfo) rev narHash lastModified;
};
blank = {
type = "github";
owner = "divnix";
repo = "blank";
inherit (depot.inputs.blank.sourceInfo) rev narHash lastModified;
};
} // import ./extra-flakes.nix;
in builtins.toJSON {
version = 2;
flakes = lib.pipe flakes [
(lib.attrsToList)
(map (f: {
from = {
type = "indirect";
id = f.name;
};
to = f.value;
}))
];
});
}

View file

@ -1,18 +0,0 @@
let
github = owner: repo: {
type = "github";
inherit owner repo;
};
in {
# own
hyprspace = github "hyprspace" "hyprspace";
ai = github "nixified-ai" "flake";
nix-super = github "privatevoid-net" "nix-super";
nixpak = github "nixpak" "nixpak";
# other
nix = github "NixOS" "nix";
flake-parts = github "hercules-ci" "flake-parts";
home-manager = github "nix-community" "home-manager";
dream2nix = github "nix-community" "dream2nix";
}

View file

@ -4,12 +4,16 @@
services.forge = {
nodes.server = [ "VEGAS" ];
nixos.server = ./server.nix;
meshLinks.server.forge.link.protocol = "http";
meshLinks.server = {
name = "forge";
link.protocol = "http";
};
secrets = with config.services.forge.nodes; {
oidcSecret = {
nodes = server;
owner = "forgejo";
};
dbCredentials.nodes = server;
};
};
@ -19,14 +23,6 @@
forge.target = config.hostLinks.${host}.forge.url;
};
patroni = config.lib.forService "forge" {
databases.forge = {};
users.forge.locksmith = {
nodes = config.services.forge.nodes.server;
format = "raw";
};
};
garage = config.lib.forService "forge" {
keys.forgejo.locksmith.nodes = config.services.forge.nodes.server;
buckets.forgejo.allow.forgejo = [ "read" "write" ];

View file

@ -26,7 +26,6 @@ in
services.locksmith.waitForSecrets.forgejo = [
"garage-forgejo-id"
"garage-forgejo-secret"
"patroni-forge"
];
services.forgejo = {
@ -40,7 +39,7 @@ in
inherit (patroni) port;
name = "forge";
user = "forge";
passwordFile = "/run/locksmith/patroni-forge";
passwordFile = secrets.dbCredentials.path;
};
settings = {
DEFAULT = {

View file

@ -0,0 +1,10 @@
{ depot, ... }:
{
services.gitlab = {
nodes.host = [ "VEGAS" ];
nixos.host = ./host.nix;
};
dns.records.git.target = [ depot.hours.VEGAS.interfaces.primary.addrPublic ];
}

View file

@ -0,0 +1,94 @@
{ cluster, config, lib, depot, ... }:
let
inherit (depot.lib.meta) domain adminEmail;
patroni = cluster.config.links.patroni-pg-access;
mkSecret = name: {
owner = "gitlab";
group = "gitlab";
mode = "0400";
file = ../../../secrets/${name}.age;
};
secrets = lib.mapAttrs (_: v: v.path) config.age.secrets;
cfg = config.services.gitlab;
in
{
age.secrets = lib.flip lib.genAttrs mkSecret [
"gitlab-db-credentials"
"gitlab-initial-root-password"
"gitlab-openid-secret"
"gitlab-secret-db"
"gitlab-secret-jws"
"gitlab-secret-otp"
"gitlab-secret-secret"
];
services.gitlab = {
enable = true;
https = true;
host = "git.${domain}";
port = 443;
databaseCreateLocally = false;
databaseHost = patroni.ipv4;
extraDatabaseConfig = { inherit (patroni) port; };
databaseUsername = "gitlab";
databasePasswordFile = secrets.gitlab-db-credentials;
initialRootEmail = adminEmail;
statePath = "/srv/storage/private/gitlab/state";
smtp = {
enable = true;
inherit domain;
};
initialRootPasswordFile = secrets.gitlab-initial-root-password;
secrets = with secrets; {
dbFile = gitlab-secret-db;
jwsFile = gitlab-secret-jws;
otpFile = gitlab-secret-otp;
secretFile = gitlab-secret-secret;
};
extraConfig = {
omniauth = {
enabled = true;
auto_sign_in_with_provider = "openid_connect";
allow_single_sign_on = ["openid_connect"];
block_auto_created_users = false;
providers = [
{
name = "openid_connect";
label = "Private Void Account";
args = {
name = "openid_connect";
scope = ["openid" "profile"];
response_type = "code";
issuer = "https://login.${domain}/auth/realms/master";
discovery = true;
client_auth_method = "query";
uid_field = "preferred_username";
client_options = {
identifier = "net.privatevoid.git2";
secret = { _secret = secrets.gitlab-openid-secret; };
redirect_uri = "https://${cfg.host}/users/auth/openid_connect/callback";
};
};
}
];
};
};
};
services.nginx.virtualHosts."${cfg.host}" = depot.lib.nginx.vhosts.proxy "http://unix:/run/gitlab/gitlab-workhorse.socket";
}

View file

@ -1,4 +1,4 @@
{ cluster, config, lib, pkgs, utils, ... }:
{ cluster, config, pkgs, utils, ... }:
let
frontendLink = cluster.config.links.idm;
@ -39,8 +39,9 @@ in
security = {
pam.services.sudo = { config, ... }: {
rules.auth.rssh = {
enable = lib.mkForce true;
order = config.rules.auth.unix.order - 10;
control = "sufficient";
modulePath = "${pkgs.pam_rssh}/lib/libpam_rssh.so";
settings = {
authorized_keys_command = "/etc/ssh/authorized_keys_command_kanidm";
authorized_keys_command_user = "nobody";

View file

@ -33,9 +33,6 @@ in
ldapbindaddress = "${ldapLink.ipv4}:${ldapLink.portStr}";
origin = frontendLink.url;
inherit domain;
online_backup = {
versions = 7;
};
};
};

View file

@ -29,7 +29,10 @@
io-tweaks = [ "VEGAS" ];
remote-api = [ "VEGAS" ];
};
meshLinks.gateway.ipfsGateway.link.protocol = "http";
meshLinks.gateway = {
name = "ipfsGateway";
link.protocol = "http";
};
nixos = {
node = [
./node.nix

View file

@ -9,7 +9,7 @@ in
environment = {
OTEL_TRACES_EXPORTER = "otlp";
OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
OTEL_EXPORTER_OTLP_ENDPOINT = "${cluster.config.ways.ingest-traces-otlp.url}:443";
OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
OTEL_TRACES_SAMPLER_ARG = "0.50";
};

View file

@ -26,7 +26,7 @@ in {
name = "logging";
positions.filename = "\${STATE_DIRECTORY:/tmp}/logging-positions.yaml";
clients = singleton {
url = "${cluster.config.ways.ingest-logs.url}/loki/api/v1/push";
url = "${cluster.config.ways.monitoring-logs.url}/loki/api/v1/push";
};
scrape_configs = singleton {
job_name = "journal";

View file

@ -18,6 +18,26 @@ in
protocol = "http";
ipv4 = meshIpFor "server";
};
tempo = {
protocol = "http";
ipv4 = meshIpFor "server";
};
tempo-grpc = {
protocol = "http";
ipv4 = "127.0.0.1";
};
tempo-otlp-http = {
protocol = "http";
ipv4 = meshIpFor "server";
};
tempo-otlp-grpc = {
protocol = "http";
ipv4 = meshIpFor "server";
};
tempo-zipkin-http = {
protocol = "http";
ipv4 = meshIpFor "server";
};
};
hostLinks = lib.genAttrs config.services.monitoring.nodes.grafana (name: {
grafana = {
@ -31,7 +51,6 @@ in
blackbox = [ "checkmate" "grail" "prophet" ];
grafana = [ "VEGAS" "prophet" ];
logging = [ "VEGAS" "grail" ];
tracing = [ "VEGAS" "grail" ];
server = [ "VEGAS" ];
};
nixos = {
@ -42,19 +61,14 @@ in
./provisioning/dashboards.nix
];
logging = ./logging.nix;
tracing = ./tracing.nix;
server = [
./server.nix
./tracing.nix
];
};
meshLinks = {
logging.loki.link.protocol = "http";
tracing = {
tempo.link.protocol = "http";
tempo-otlp-http.link.protocol = "http";
tempo-otlp-grpc.link.protocol = "grpc";
tempo-zipkin-http.link.protocol = "http";
};
meshLinks.logging = {
name = "loki";
link.protocol = "http";
};
};
@ -68,51 +82,29 @@ in
nodes = config.services.monitoring.nodes.logging;
format = "envFile";
};
tempo-ingest.locksmith = {
nodes = config.services.monitoring.nodes.tracing;
format = "envFile";
};
tempo-query.locksmith = {
nodes = config.services.monitoring.nodes.tracing;
format = "envFile";
};
tempo = { };
};
buckets = {
loki-chunks.allow = {
loki-ingest = [ "read" "write" ];
loki-query = [ "read" ];
};
tempo-chunks.allow = {
tempo-ingest = [ "read" "write" ];
tempo-query = [ "read" ];
};
tempo-chunks.allow.tempo = [ "read" "write" ];
};
};
ways = let
query = consulService: {
inherit consulService;
internal = true;
extras.extraConfig = ''
proxy_read_timeout 3600s;
'';
ways = config.lib.forService "monitoring" {
monitoring = {
consulService = "grafana";
extras.locations."/".proxyWebsockets = true;
};
ingest = consulService: {
inherit consulService;
monitoring-logs = {
internal = true;
consulService = "loki";
extras.extraConfig = ''
client_max_body_size 4G;
proxy_read_timeout 3600s;
'';
};
in config.lib.forService "monitoring" {
monitoring = {
consulService = "grafana";
extras.locations."/".proxyWebsockets = true;
};
monitoring-logs = query "loki";
monitoring-traces = query "tempo";
ingest-logs = ingest "loki";
ingest-traces-otlp = ingest "tempo-ingest-otlp-grpc" // { grpc = true; };
};
}

View file

@ -73,16 +73,6 @@ in
inherit (cluster.config.ways.monitoring-logs) url;
type = "loki";
}
{
name = "Tempo";
uid = "P214B5B846CF3925F";
inherit (cluster.config.ways.monitoring-traces) url;
type = "tempo";
jsonData = {
serviceMap.datasourceUid = "PBFA97CFB590B2093";
nodeGraph.enabled = true;
};
}
];
};
};

View file

@ -19,23 +19,10 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1210,
"id": 16,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 7,
"panels": [],
"title": "Replication",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
@ -71,7 +58,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 1
"y": 0
},
"id": 2,
"options": {
@ -79,7 +66,6 @@
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -87,11 +73,9 @@
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
"textMode": "auto"
},
"pluginVersion": "11.1.3",
"pluginVersion": "9.5.1",
"targets": [
{
"datasource": {
@ -154,7 +138,7 @@
"h": 8,
"w": 4,
"x": 12,
"y": 1
"y": 0
},
"id": 3,
"options": {
@ -162,17 +146,14 @@
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [],
"fields": "/^instance$/",
"values": true
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
"textMode": "auto"
},
"pluginVersion": "11.1.3",
"pluginVersion": "9.5.1",
"targets": [
{
"datasource": {
@ -217,7 +198,7 @@
"h": 8,
"w": 8,
"x": 16,
"y": 1
"y": 0
},
"id": 5,
"options": {
@ -273,7 +254,6 @@
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
@ -287,7 +267,6 @@
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
@ -325,7 +304,7 @@
"h": 13,
"w": 12,
"x": 0,
"y": 9
"y": 8
},
"id": 1,
"options": {
@ -369,7 +348,6 @@
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
@ -383,7 +361,6 @@
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
@ -421,7 +398,7 @@
"h": 13,
"w": 12,
"x": 12,
"y": 9
"y": 8
},
"id": 4,
"options": {
@ -451,326 +428,24 @@
],
"title": "Activity",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 22
},
"id": 8,
"panels": [],
"title": "Storage",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 8,
"x": 0,
"y": 23
},
"id": 6,
"options": {
"displayLabels": [
"percent"
],
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Value",
"sortDesc": true,
"values": [
"percent",
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.1.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "max by(datname) (sum by(datname, instance) (pg_database_size_bytes))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A",
"useBackend": false
}
],
"title": "Database Size",
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 25,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 5,
"x": 8,
"y": 23
},
"id": 9,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.1.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "pg_wal_size_bytes",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{instance}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "WAL Size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 25,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 3,
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 11,
"x": 13,
"y": 23
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "max(sum by(instance) (pg_database_size_bytes))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Total DB Size",
"type": "timeseries"
}
],
"refresh": "5m",
"schemaVersion": 39,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "PostgreSQL HA",
"uid": "dc7545ef-3180-4a5e-a289-1e64571ebb87",
"version": 2,
"version": 7,
"weekStart": ""
},
"folderId": 0,

View file

@ -0,0 +1,16 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A KhCGp7PAMGrEdzRxBrsW4tRk30JwpI+4lPzrRCUhSw4
8s7WqA5c3zS1euN5R+jfFNBdvr8OQW8P4NFeqtNsIKo
-> ssh-ed25519 5/zT0w 79hJQ2H76EZTW7YcQFCtKaS5Kbssx4Z8dPFjIVzRgFk
A1fDJbUnyIRy+kWa3PhJNj/SdRPlcEy6FYsAfnuZ2AQ
-> ssh-ed25519 d3WGuA aylkdL1KliM1NfrYDGlG8X6YjXvVUCU4sV90I+a840U
6sXdqIPjtoNSylZRh1DCghHOwDo+fC7WB4QWQoWmG48
-> //gd+2-grease baUWA$3 z-qs3W O/2.1W
Sfq3+rkMJhpUTTmcos5TaaUtX2Ip9pciHAZLiWPix+C9N7ccac/1W5RNedMJCLsq
MQ+xKzexf8+hgNVhKOksvbKBBROXqk1bUOKk8w3OgFPmmByzmCBUwkdkeu5DFTYR
rg
--- kUl1uIPRkM5y7C68kdN22pMKXP7gazyha4PE+ap0Jqw
w>×Àè¥
<15>CÈ,\‰ßœI¯ˆúHxG@^Çá“På ÃþÙÏlw6µŽ{þ’rbé5æ†T>Êñ
ÚWܤX4Kp(ß?9ˆß­^^oP3f </v3N$ê¤sÓbŽ¾> O™÷œ+òN0άïµàDtêŽ5Vº#è ¶³ îŸ#y|@ŒGzSi»­ô*·HùüŽ]
ꎀ5

View file

@ -1,16 +1,14 @@
{ cluster, config, pkgs, ... }:
let
inherit (cluster.config.links) prometheus-ingest;
inherit (config.links) tempo-grpc;
links = cluster.config.hostLinks.${config.networking.hostName};
inherit (cluster.config) links;
dataDir = "/srv/storage/private/tempo";
tempoConfig = {
server = {
http_listen_address = links.tempo.ipv4;
http_listen_port = links.tempo.port;
grpc_listen_address = tempo-grpc.ipv4;
grpc_listen_port = tempo-grpc.port;
grpc_listen_address = links.tempo-grpc.ipv4;
grpc_listen_port = links.tempo-grpc.port;
};
distributor.receivers = {
otlp = {
@ -21,7 +19,7 @@ let
};
zipkin.endpoint = links.tempo-zipkin-http.tuple;
};
querier.frontend_worker.frontend_address = tempo-grpc.tuple;
querier.frontend_worker.frontend_address = links.tempo-grpc.tuple;
ingester = {
trace_idle_period = "30s";
max_block_bytes = 1000000;
@ -58,7 +56,7 @@ let
path = "${dataDir}/generator/wal";
remote_write = [
{
url = "${prometheus-ingest.url}/api/v1/write";
url = "${links.prometheus-ingest.url}/api/v1/write";
send_exemplars = true;
}
];
@ -70,11 +68,7 @@ let
];
};
in {
links.tempo-grpc.protocol = "http";
services.locksmith.waitForSecrets.tempo = [
"garage-tempo-ingest"
];
age.secrets.tempoSecrets.file = ./secrets/tempo-secrets.age;
users.users.tempo = {
isSystemUser = true;
@ -87,53 +81,24 @@ in {
systemd.services.tempo = {
wantedBy = [ "multi-user.target" ];
distributed = {
enable = true;
registerServices = [
"tempo"
"tempo-ingest-otlp-grpc"
];
};
serviceConfig = {
User = "tempo";
Group = "tempo";
ExecStart = "${pkgs.tempo}/bin/tempo -config.file=${pkgs.writeText "tempo.yaml" (builtins.toJSON tempoConfig)}";
PrivateTmp = true;
EnvironmentFile = "/run/locksmith/garage-tempo-ingest";
EnvironmentFile = config.age.secrets.tempoSecrets.path;
};
};
consul.services = {
tempo = {
mode = "manual";
definition = {
name = "tempo";
address = links.tempo.ipv4;
inherit (links.tempo) port;
checks = [
{
name = "Tempo";
id = "service:tempo:backend";
interval = "5s";
http = "${links.tempo.url}/ready";
}
];
services.grafana.provision.datasources.settings.datasources = [
{
name = "Tempo";
uid = "P214B5B846CF3925F";
inherit (links.tempo) url;
type = "tempo";
jsonData = {
serviceMap.datasourceUid = "PBFA97CFB590B2093"; # prometheus
nodeGraph.enabled = true;
};
};
tempo-ingest-otlp-grpc = {
mode = "manual";
definition = {
name = "tempo-ingest-otlp-grpc";
address = links.tempo-otlp-grpc.ipv4;
inherit (links.tempo-otlp-grpc) port;
checks = [
{
name = "Tempo Service Status";
id = "service:tempo-ingest-otlp-grpc:tempo";
alias_service = "tempo";
}
];
};
};
};
}
];
}

View file

@ -18,7 +18,7 @@ in
};
};
services.nextcloud = {
package = pkgs.nextcloud30;
package = pkgs.nextcloud29;
enable = true;
https = true;
hostName = "storage.${depot.lib.meta.domain}";

View file

@ -1,4 +1,4 @@
{ config, depot, ... }:
{ depot, ... }:
{
services.sso = {
@ -18,12 +18,4 @@
login.target = ssoAddr;
account.target = ssoAddr;
};
patroni = config.lib.forService "sso" {
databases.keycloak = {};
users.keycloak.locksmith = {
nodes = config.services.sso.nodes.host;
format = "raw";
};
};
}

View file

@ -8,10 +8,12 @@ in
{
links.keycloak.protocol = "http";
services.locksmith.waitForSecrets.keycloak = [
"patroni-keycloak"
];
age.secrets.keycloak-dbpass = {
file = ../../../secrets/keycloak-dbpass.age;
owner = "root";
group = "root";
mode = "0400";
};
services.nginx.virtualHosts = {
"${login}" = lib.recursiveUpdate (vhosts.proxy kc.url) {
locations = {
@ -34,14 +36,13 @@ in
host = patroni.ipv4;
inherit (patroni) port;
useSSL = false;
passwordFile = "/run/locksmith/patroni-keycloak";
passwordFile = config.age.secrets.keycloak-dbpass.path;
};
settings = {
http-enabled = true;
http-host = kc.ipv4;
http-port = kc.port;
hostname = login;
proxy-headers = "xforwarded";
proxy = "edge";
# for backcompat, TODO: remove
http-relative-path = "/auth";
};
@ -53,7 +54,7 @@ in
"-Dotel.traces.exporter=otlp"
];
OTEL_EXPORTER_OTLP_PROTOCOL = "grpc";
OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.ways.ingest-traces-otlp.url;
OTEL_EXPORTER_OTLP_ENDPOINT = cluster.config.links.tempo-otlp-grpc.url;
OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
OTEL_TRACES_SAMPLER_ARG = "0.50";
};

View file

@ -56,7 +56,7 @@ in
};
simulacrum = {
enable = true;
deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" "ways" ];
deps = [ "wireguard" "consul" "locksmith" "dns" "incandescence" ];
settings = ./simulacrum/test.nix;
};
};

View file

@ -36,9 +36,10 @@ in
inherit (linkWeb) port;
checks = [
{
name = "Garage Service Status";
id = "service:garage-web:garage";
alias_service = "garage";
name = "Garage Node";
id = "service:garage-web:node";
interval = "5s";
http = "${config.links.garageMetrics.url}/health";
}
];
};

View file

@ -1,15 +1,11 @@
{ cluster, config, depot, lib, pkgs, ... }:
{ cluster, config, lib, pkgs, ... }:
let
externalWays = lib.filterAttrs (_: cfg: !cfg.internal) cluster.config.ways;
internalWays = lib.filterAttrs (_: cfg: cfg.internal) cluster.config.ways;
byMode = lib.pipe cluster.config.ways [
(lib.attrsToList)
(lib.groupBy (way: way.value.mode))
(lib.mapAttrs (n: v: lib.listToAttrs v))
];
consulServiceWays = lib.filterAttrs (_: cfg: cfg.useConsul) cluster.config.ways;
in
{
@ -29,20 +25,7 @@ in
];
locations = lib.mkMerge [
{
"/" = if cfg.mode == "static" then {
root = cfg.static {
inherit depot;
inherit pkgs;
inherit (pkgs) system;
};
} else if cfg.grpc then {
extraConfig = ''
set $nix_proxy_grpc_target ${cfg.target};
grpc_pass $nix_proxy_grpc_target;
'';
} else {
proxyPass = cfg.target;
};
"/".proxyPass = cfg.target;
"${cfg.healthCheckPath}".extraConfig = "access_log off;";
}
{
@ -57,7 +40,7 @@ in
};
}) cluster.config.ways;
appendHttpConfig = lib.mkIf (byMode.consul != {}) ''
appendHttpConfig = lib.mkIf (consulServiceWays != {}) ''
include /run/consul-template/nginx-ways-*.conf;
'';
};
@ -77,7 +60,7 @@ in
value.distributed.enable = true;
}) externalWays;
services.consul-template.instances.ways = lib.mkIf (byMode.consul != {}) {
services.consul-template.instances.ways = lib.mkIf (consulServiceWays != {}) {
user = "nginx";
group = "nginx";
settings = {
@ -96,14 +79,14 @@ in
{{ else }}
# upstream ${cfg.nginxUpstreamName} (${cfg.consulService}): no servers available
{{ end }}
'') byMode.consul;
'') consulServiceWays;
in pkgs.writeText "ways-upstreams.ctmpl" (lib.concatStringsSep "\n" (lib.unique upstreams));
destination = "/run/consul-template/nginx-ways-upstreams.conf";
exec.command = lib.singleton (pkgs.writeShellScript "ways-reload" ''
if ${config.systemd.package}/bin/systemctl is-active nginx.service; then
exec ${config.services.nginx.package}/bin/nginx -s reload -g 'pid /run/nginx/nginx.pid;'
fi
'');
exec.command = [
"${config.services.nginx.package}/bin/nginx"
"-s" "reload"
"-g" "pid /run/nginx/nginx.pid;"
];
}
];
};

View file

@ -35,12 +35,6 @@ with lib;
};
};
grpc = mkOption {
description = "Whether this endpoint is a gRPC service.";
type = types.bool;
default = false;
};
target = mkOption {
type = types.str;
};
@ -58,10 +52,6 @@ with lib;
type = types.str;
};
static = mkOption {
type = with types; functionTo (coercedTo package (package: "${package.webroot or package}") str);
};
healthCheckPath = mkOption {
type = types.path;
default = "/.well-known/ways/internal-health-check";
@ -73,10 +63,10 @@ with lib;
default = "https://${name}.${config.domainSuffix}";
};
mode = mkOption {
type = types.enum [ "simple" "consul" "static" ];
useConsul = mkOption {
type = types.bool;
internal = true;
default = "simple";
default = false;
};
nginxUpstreamName = mkOption {
@ -109,15 +99,12 @@ with lib;
config = lib.mkMerge [
(lib.mkIf options.consulService.isDefined {
mode = "consul";
useConsul = true;
nginxUpstreamName = "ways_upstream_${builtins.hashString "md5" options.consulService.value}";
target = "${if config.grpc then "grpc" else "http"}://${options.nginxUpstreamName.value}";
target = "http://${options.nginxUpstreamName.value}";
})
(lib.mkIf options.bucket.isDefined {
consulService = "garage-web";
})
(lib.mkIf options.static.isDefined {
mode = "static";
})
];
}

View file

@ -10,11 +10,11 @@
"systems": "systems"
},
"locked": {
"lastModified": 1723293904,
"narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
"lastModified": 1722339003,
"narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
"owner": "ryantm",
"repo": "agenix",
"rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
"rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
"type": "github"
},
"original": {
@ -29,8 +29,9 @@
"flake-compat": [
"blank"
],
"flake-parts": "flake-parts",
"nix-github-actions": "nix-github-actions",
"flake-utils": [
"repin-flake-utils"
],
"nixpkgs": [
"nixpkgs"
],
@ -39,11 +40,11 @@
]
},
"locked": {
"lastModified": 1730906442,
"narHash": "sha256-tBuyb8jWBSHHgcIrOfiyQJZGY1IviMzH2V74t7gWfgI=",
"lastModified": 1722472866,
"narHash": "sha256-GJIz4M5HDB948Ex/8cPvbkrNzl/eKUE7/c21JBu4lb8=",
"owner": "zhaofengli",
"repo": "attic",
"rev": "d0b66cf897e4d55f03d341562c9821dc4e566e54",
"rev": "e127acbf9a71ebc0c26bc8e28346822e0a6e16ba",
"type": "github"
},
"original": {
@ -75,11 +76,11 @@
]
},
"locked": {
"lastModified": 1722960479,
"narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
"lastModified": 1717025063,
"narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
"owner": "ipetkov",
"repo": "crane",
"rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
"rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
"type": "github"
},
"original": {
@ -117,11 +118,11 @@
]
},
"locked": {
"lastModified": 1728330715,
"narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
"lastModified": 1722113426,
"narHash": "sha256-Yo/3loq572A8Su6aY5GP56knpuKYRvM2a1meP9oJZCw=",
"owner": "numtide",
"repo": "devshell",
"rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
"rev": "67cce7359e4cd3c45296fb4aaf6a19e2a9c757ae",
"type": "github"
},
"original": {
@ -173,11 +174,11 @@
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1696426674,
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
@ -189,7 +190,6 @@
"flake-parts": {
"inputs": {
"nixpkgs-lib": [
"attic",
"nixpkgs"
]
},
@ -210,15 +210,16 @@
"flake-parts_2": {
"inputs": {
"nixpkgs-lib": [
"nix-super",
"nixpkgs"
]
},
"locked": {
"lastModified": 1730504689,
"narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
"lastModified": 1712014858,
"narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "506278e768c2a08bec68eb62932193e341f55c90",
"rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
"type": "github"
},
"original": {
@ -227,71 +228,18 @@
"type": "github"
}
},
"flake-parts_3": {
"inputs": {
"nixpkgs-lib": [
"nix-super",
"nixpkgs"
]
},
"flake-utils": {
"locked": {
"lastModified": 1719994518,
"narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"git-hooks-nix": {
"inputs": {
"flake-compat": [
"nix-super"
],
"gitignore": [
"nix-super"
],
"nixpkgs": [
"nix-super",
"nixpkgs"
],
"nixpkgs-stable": [
"nix-super",
"nixpkgs"
]
},
"locked": {
"lastModified": 1721042469,
"narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "git-hooks.nix",
"type": "github"
}
},
"haskell-flake": {
"locked": {
"lastModified": 1684780604,
"narHash": "sha256-2uMZsewmRn7rRtAnnQNw1lj0uZBMh4m6Cs/7dV5YF08=",
"owner": "srid",
"repo": "haskell-flake",
"rev": "74210fa80a49f1b6f67223debdbf1494596ff9f2",
"type": "github"
},
"original": {
"owner": "srid",
"ref": "0.3.0",
"repo": "haskell-flake",
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
@ -300,19 +248,19 @@
"flake-parts": [
"flake-parts"
],
"haskell-flake": "haskell-flake",
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1723736589,
"narHash": "sha256-/Vdg5ZKtP71ZEKVV6JXlrOEu0CM2Flcs+nwDmWRzgjQ=",
"owner": "hercules-ci",
"lastModified": 1723760507,
"narHash": "sha256-Dx4kNqDE3//22xyICJ6czV465MUvspowZJlLGSbSGpo=",
"owner": "nrabulinski",
"repo": "hercules-ci-agent",
"rev": "c303cc8e437c0fd26b9452472e7df5aa374e9177",
"rev": "c41b58bf33c9167670685eb97ede9ac240b321d4",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"owner": "nrabulinski",
"ref": "ifdless",
"repo": "hercules-ci-agent",
"type": "github"
}
@ -327,11 +275,11 @@
]
},
"locked": {
"lastModified": 1730903510,
"narHash": "sha256-mnynlrPeiW0nUQ8KGZHb3WyxAxA3Ye/BH8gMjdoKP6E=",
"lastModified": 1719226092,
"narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=",
"owner": "hercules-ci",
"repo": "hercules-ci-effects",
"rev": "b89ac4d66d618b915b1f0a408e2775fe3821d141",
"rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5",
"type": "github"
},
"original": {
@ -385,16 +333,15 @@
"libgit2": {
"flake": false,
"locked": {
"lastModified": 1715853528,
"narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
"lastModified": 1697646580,
"narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
"owner": "libgit2",
"repo": "libgit2",
"rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
"rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
"type": "github"
},
"original": {
"owner": "libgit2",
"ref": "v1.8.1",
"repo": "libgit2",
"type": "github"
}
@ -436,11 +383,11 @@
"systems": "systems_2"
},
"locked": {
"lastModified": 1725623828,
"narHash": "sha256-5Zrn72PO9yBaNO4Gd5uOsEmRpYH5rVAFKOQ5h2PxyhU=",
"lastModified": 1722409392,
"narHash": "sha256-8QuMS00EutmqzAIPxyJEPxM8EHiWlSKs6E2Htoh3Kes=",
"owner": "numtide",
"repo": "nar-serve",
"rev": "e5c749a444f2d14f381c75ef3a8feaa82c333b92",
"rev": "9d0eff868d328fe67c60c26c8ba50e0b9d8de867",
"type": "github"
},
"original": {
@ -451,11 +398,11 @@
},
"nix-filter": {
"locked": {
"lastModified": 1730207686,
"narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
"lastModified": 1710156097,
"narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
"owner": "numtide",
"repo": "nix-filter",
"rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
"rev": "3342559a24e85fc164b295c3444e8a139924675b",
"type": "github"
},
"original": {
@ -464,51 +411,31 @@
"type": "github"
}
},
"nix-github-actions": {
"inputs": {
"nixpkgs": [
"attic",
"nixpkgs"
]
},
"locked": {
"lastModified": 1729742964,
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nix-github-actions",
"type": "github"
}
},
"nix-super": {
"inputs": {
"flake-compat": "flake-compat_2",
"flake-parts": "flake-parts_3",
"git-hooks-nix": "git-hooks-nix",
"flake-parts": "flake-parts_2",
"libgit2": "libgit2",
"nixpkgs": "nixpkgs_3",
"nixpkgs-23-11": [
"blank"
],
"nixpkgs-regression": [
"blank"
]
],
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 1731271232,
"narHash": "sha256-HSNUAfhtG/A2hnrzPMT3asZZ2Wb3dAmedOr1VzptOCg=",
"rev": "1eb19dd804a83d99c497118af8ab781eee569c65",
"type": "tarball",
"url": "https://forge.privatevoid.net/api/v1/repos/max/nix-super/archive/1eb19dd804a83d99c497118af8ab781eee569c65.tar.gz"
"host": "git.privatevoid.net",
"lastModified": 1713821351,
"narHash": "sha256-JctHGT1oa4pet4PgUKRM7pf0w+qGe0a/ahVij8bee3o=",
"owner": "max",
"repo": "nix-super",
"rev": "5ecd820c18b1aaa3c8ee257a7a9a2624c4107031",
"type": "gitlab"
},
"original": {
"type": "tarball",
"url": "https://forge.privatevoid.net/max/nix-super/archive/master.tar.gz"
"host": "git.privatevoid.net",
"owner": "max",
"repo": "nix-super",
"type": "gitlab"
}
},
"nixpkgs": {
@ -545,27 +472,27 @@
},
"nixpkgs_3": {
"locked": {
"lastModified": 1723688146,
"narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=",
"lastModified": 1709083642,
"narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c3d4ac725177c030b1e289015989da2ad9d56af0",
"rev": "b550fe4b4776908ac2a861124307045f8e717c8e",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-24.05",
"ref": "release-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_4": {
"locked": {
"lastModified": 1730785428,
"narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
"lastModified": 1722539632,
"narHash": "sha256-g4L+I8rDl7RQy5x8XcEMqNO49LFhrHTzVBqXtG2+FGo=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
"rev": "f2d6c7123138044e0c68902268bd8f37dd7e2fa7",
"type": "github"
},
"original": {
@ -575,6 +502,38 @@
"type": "github"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-compat": [
"nix-super"
],
"flake-utils": "flake-utils",
"gitignore": [
"nix-super"
],
"nixpkgs": [
"nix-super",
"nixpkgs"
],
"nixpkgs-stable": [
"nix-super",
"nixpkgs"
]
},
"locked": {
"lastModified": 1712897695,
"narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"repin-flake-utils": {
"inputs": {
"systems": [
@ -582,11 +541,11 @@
]
},
"locked": {
"lastModified": 1726560853,
"narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
"lastModified": 1710146030,
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
"type": "github"
},
"original": {
@ -602,7 +561,7 @@
"blank": "blank",
"devshell": "devshell",
"drv-parts": "drv-parts",
"flake-parts": "flake-parts_2",
"flake-parts": "flake-parts",
"hercules-ci-agent": "hercules-ci-agent",
"hercules-ci-effects": "hercules-ci-effects",
"hyprspace": "hyprspace",

View file

@ -30,10 +30,9 @@
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
nix-super = {
url = "https://forge.privatevoid.net/max/nix-super/archive/master.tar.gz";
url = "gitlab:max/nix-super?host=git.privatevoid.net";
inputs = {
nixpkgs-regression.follows = "blank";
nixpkgs-23-11.follows = "blank";
};
};
@ -55,6 +54,7 @@
nixpkgs.follows = "nixpkgs";
nixpkgs-stable.follows = "nixpkgs";
flake-compat.follows = "blank";
flake-utils.follows = "repin-flake-utils";
};
};
@ -83,7 +83,7 @@
};
hercules-ci-agent = {
url = "github:hercules-ci/hercules-ci-agent";
url = "github:nrabulinski/hercules-ci-agent/ifdless";
inputs = {
flake-parts.follows = "flake-parts";
};

View file

@ -26,13 +26,11 @@ in
cfg = v.distributed;
svcs = map (x: config.consul.services.${x}) cfg.registerServices;
svc = config.consul.services.${cfg.registerService};
runWithRegistration = pkgs.writeShellScript "run-with-registration" ''
trap '${lib.concatStringsSep ";" (map (svc: svc.commands.deregister) svcs)}' EXIT
${lib.concatStringsSep "\n" (
map (svc: svc.commands.register) svcs
)}
trap '${svc.commands.deregister}' EXIT
${svc.commands.register}
''${@}
'';
@ -51,10 +49,10 @@ in
[Service]
ExecStartPre=${waitForConsul} 'services/${n}%i'
ExecStart=
ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerServices != []) runWithRegistration} ${ExecStart}
ExecStart=${consul}/bin/consul lock --name=${n} --n=${toString cfg.replicas} --shell=false --child-exit-code 'services/${n}%i' ${optionalString (cfg.registerService != null) runWithRegistration} ${ExecStart}
Environment="CONSUL_HTTP_ADDR=${consulHttpAddr}"
${optionalString (v.serviceConfig ? RestrictAddressFamilies) "RestrictAddressFamilies=AF_NETLINK"}
${optionalString (cfg.registerServices != []) (lib.concatStringsSep "\n" (map (svc: "ExecStopPost=${svc.commands.deregister}") svcs))}
${optionalString (cfg.registerService != null) "ExecStopPost=${svc.commands.deregister}"}
''))
];
}

View file

@ -14,7 +14,7 @@
experimental-features = nix-command flakes cgroups
use-cgroups = true
builders-use-substitutes = true
flake-registry = https://registry.${depot.lib.meta.domain}/flake-registry.json
flake-registry = https://git.${depot.lib.meta.domain}/private-void/registry/-/raw/master/registry.json
# For Hercules CI agent
narinfo-cache-negative-ttl = 0

View file

@ -17,11 +17,6 @@ with lib;
type = with types; nullOr str;
default = null;
};
registerServices = mkOption {
description = "Consul services to register when this service gets started.";
type = with types; listOf str;
default = if config.distributed.registerService == null then [ ] else [ config.distributed.registerService ];
};
};
}));
};

View file

@ -7,8 +7,7 @@ nixosTest {
package = keycloak;
database.passwordFile = builtins.toFile "keycloak-test-password" "kcnixostest1234";
settings = {
http-enabled = true;
proxy-headers = "xforwarded";
proxy = "edge";
hostname = "keycloak.local";
};
};

View file

@ -18,10 +18,6 @@ super: rec {
};
};
jitsi-meet-insecure = let
olm-insecure = assert builtins.length super.olm.meta.knownVulnerabilities > 0; super.olm.overrideAttrs (o: { meta = o.meta // { knownVulnerabilities = []; }; });
in super.jitsi-meet.override { olm = olm-insecure; };
jre17_standard = let
jre = super.jre_minimal.override {
jdk = super.jdk17_headless;
@ -52,6 +48,7 @@ super: rec {
s3ql = (patch super.s3ql "patches/base/s3ql").overrideAttrs (old: {
propagatedBuildInputs = old.propagatedBuildInputs ++ [
super.python3Packages.packaging
super.python3Packages.systemd
];
});

View file

@ -9,9 +9,7 @@
options.shadows = lib.mkOption {
type = with lib.types; lazyAttrsOf package;
default = {
jitsi-meet = self'.packages.jitsi-meet-insecure;
};
default = { };
};
};
}

View file

@ -1,28 +1,25 @@
diff --git a/unix_integration/resolver/src/idprovider/kanidm.rs b/unix_integration/resolver/src/idprovider/kanidm.rs
index 63cedb4d5..35c45fb0e 100644
--- a/unix_integration/resolver/src/idprovider/kanidm.rs
+++ b/unix_integration/resolver/src/idprovider/kanidm.rs
@@ -7,6 +7,7 @@ use kanidm_proto::internal::OperationError;
diff --git a/unix_integration/src/idprovider/kanidm.rs b/unix_integration/src/idprovider/kanidm.rs
index 6fc015756..31593f03e 100644
--- a/unix_integration/src/idprovider/kanidm.rs
+++ b/unix_integration/src/idprovider/kanidm.rs
@@ -4,6 +4,7 @@ use kanidm_client::{ClientError, KanidmClient, StatusCode};
use kanidm_proto::internal::OperationError;
use kanidm_proto::v1::{UnixGroupToken, UnixUserToken};
use std::collections::BTreeSet;
use std::time::{Duration, SystemTime};
use tokio::sync::{broadcast, RwLock};
+use std::env;
use tokio::sync::{broadcast, Mutex};
use kanidm_lib_crypto::CryptoPolicy;
@@ -38,6 +39,8 @@ struct KanidmProviderInternal {
hmac_key: HmacKey,
crypto_policy: CryptoPolicy,
pam_allow_groups: BTreeSet<String>,
use super::interface::{
// KeyStore,
@@ -25,12 +26,28 @@ const TAG_IDKEY: &str = "idkey";
pub struct KanidmProvider {
client: RwLock<KanidmClient>,
+ auth_name: Option<String>,
+ auth_password: Option<String>,
}
pub struct KanidmProvider {
@@ -102,6 +105,19 @@ impl KanidmProvider {
.map(|GroupMap { local, with }| (local, Id::Name(with)))
.collect();
impl KanidmProvider {
pub fn new(client: KanidmClient) -> Self {
+ let env_username: Option<String>;
+ let env_password: Option<String>;
+ match (env::var_os("KANIDM_NAME"), env::var_os("KANIDM_PASSWORD")) {
@ -35,29 +32,23 @@ index 63cedb4d5..35c45fb0e 100644
+ env_password = None;
+ }
+ }
+
Ok(KanidmProvider {
inner: Mutex::new(KanidmProviderInternal {
state: CacheState::OfflineNextCheck(now),
@@ -109,6 +125,8 @@ impl KanidmProvider {
hmac_key,
crypto_policy,
pam_allow_groups,
+ auth_name: env_username,
+ auth_password: env_password
}),
map_group,
})
@@ -256,7 +274,11 @@ impl KanidmProviderInternal {
KanidmProvider {
client: RwLock::new(client),
+ auth_name: env_username,
+ auth_password: env_password,
}
}
}
@@ -118,7 +135,11 @@ impl IdProvider for KanidmProvider {
async fn attempt_online(&mut self, _tpm: &mut tpm::BoxedDynTpm, now: SystemTime) -> bool {
- match self.client.auth_anonymous().await {
// Needs .read on all types except re-auth.
async fn provider_authenticate(&self, _tpm: &mut tpm::BoxedDynTpm) -> Result<(), IdpError> {
- match self.client.write().await.auth_anonymous().await {
+ let auth_method = match (&self.auth_name, &self.auth_password) {
+ (Some(name), Some(password)) => self.client.auth_simple_password(name, password).await,
+ _ => self.client.auth_anonymous().await
+ (Some(name), Some(password)) => self.client.write().await.auth_simple_password(name, password).await,
+ _ => self.client.write().await.auth_anonymous().await
+ };
+ match auth_method {
Ok(_uat) => {
self.state = CacheState::Online;
true
Ok(_uat) => Ok(()),
Err(err) => {
error!(?err, "Provider authentication failed");

View file

@ -0,0 +1,18 @@
diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py
index a4b55fd1..267b9a12 100644
--- a/src/s3ql/block_cache.py
+++ b/src/s3ql/block_cache.py
@@ -86,10 +86,10 @@ class CacheEntry:
def flush(self):
self.fh.flush()
- def seek(self, off):
+ def seek(self, off, whence=0):
if self.pos != off:
- self.fh.seek(off)
- self.pos = off
+ self.fh.seek(off, whence)
+ self.pos = self.fh.tell()
def tell(self):
return self.pos

View file

@ -0,0 +1,26 @@
diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
index 6402fec1..9ed3627e 100644
--- a/src/s3ql/backends/comprenc.py
+++ b/src/s3ql/backends/comprenc.py
@@ -276,7 +276,7 @@ class ComprencBackend(AbstractBackend):
buf.seek(0)
fh = buf
- return self.backend.write_fh(key, fh, meta_raw)
+ return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
def contains(self, key):
return self.backend.contains(key)
diff --git a/src/s3ql/database.py b/src/s3ql/database.py
index bb4054e6..c2142bf6 100644
--- a/src/s3ql/database.py
+++ b/src/s3ql/database.py
@@ -659,7 +659,7 @@ def upload_metadata(
)
obj = METADATA_OBJ_NAME % (blockno, params.seq_no)
fh.seek(blockno * blocksize)
- backend.write_fh(obj, fh, len_=blocksize)
+ backend.write_fh(obj, fh, len_=min(blocksize, db_size - blockno * blocksize))
if not update_params:
return

View file

@ -0,0 +1,17 @@
diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py
index 9ed3627e..db419bb7 100644
--- a/src/s3ql/backends/comprenc.py
+++ b/src/s3ql/backends/comprenc.py
@@ -276,6 +276,12 @@ class ComprencBackend(AbstractBackend):
buf.seek(0)
fh = buf
+ if meta_raw['compression'] == 'None' and meta_raw['encryption'] == 'None':
+ buf = io.BytesIO()
+ copyfh(fh, buf, len_)
+ buf.seek(0)
+ fh = buf
+
return self.backend.write_fh(key, fh, meta_raw, len_=len_ if meta_raw['compression'] == 'None'and meta_raw['encryption'] == 'None' else None)
def contains(self, key):

View file

@ -1,13 +0,0 @@
diff --git a/src/s3ql/database.py b/src/s3ql/database.py
index 1c6df119..f3a47781 100644
--- a/src/s3ql/database.py
+++ b/src/s3ql/database.py
@@ -677,7 +677,7 @@ def upload_metadata(
)
obj = METADATA_OBJ_NAME % (blockno, params.seq_no)
fh.seek(blockno * blocksize)
- backend.write_fh(obj, fh, len_=blocksize)
+ backend.write_fh(obj, fh, len_=min(blocksize, db_size - blockno * blocksize))
if not update_params:
return

View file

@ -0,0 +1,12 @@
diff --git a/tests/t0_http.py b/tests/t0_http.py
index 66ed564f..36bebab1 100755
--- a/tests/t0_http.py
+++ b/tests/t0_http.py
@@ -289,7 +289,6 @@ def do_GET(self):
# We don't *actually* want to establish SSL, that'd be
# to complex for our mock server
- monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
conn = HTTPConnection(
test_host,
test_port,

View file

@ -0,0 +1,26 @@
commit 1edbbcf08d5701ea38f13fca7491418318aebca9
Author: Max <max@privatevoid.net>
Date: Fri Jun 7 23:31:08 2024 +0200
accurate length
diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py
index 2995ca4f..3c3c79ab 100644
--- a/src/s3ql/backends/s3c.py
+++ b/src/s3ql/backends/s3c.py
@@ -387,9 +387,13 @@ class Backend(AbstractBackend):
'''
off = fh.tell()
+ fh.seek(0, os.SEEK_END)
+ actual_len = fh.tell() - off
+ fh.seek(off, os.SEEK_SET)
if len_ is None:
- fh.seek(0, os.SEEK_END)
- len_ = fh.tell()
+ len_ = actual_len
+ else:
+ len_ = min(len_, actual_len)
return self._write_fh(key, fh, off, len_, metadata or {})
@retry

View file

@ -0,0 +1,392 @@
From 11e3a9cea77cd8498d874f7fd69a938af4da68cd Mon Sep 17 00:00:00 2001
From: xeji <36407913+xeji@users.noreply.github.com>
Date: Thu, 28 Mar 2024 22:19:11 +0100
Subject: [PATCH] new backend s3c4: s3c with V4 request signatures (#349)
---
rst/backends.rst | 15 ++++
src/s3ql/backends/__init__.py | 3 +-
src/s3ql/backends/s3.py | 100 ++----------------------
src/s3ql/backends/s3c4.py | 140 ++++++++++++++++++++++++++++++++++
src/s3ql/parse_args.py | 2 +-
tests/mock_server.py | 11 +++
6 files changed, 174 insertions(+), 97 deletions(-)
create mode 100644 src/s3ql/backends/s3c4.py
diff --git a/rst/backends.rst b/rst/backends.rst
index 7220ee96..4bc68387 100644
--- a/rst/backends.rst
+++ b/rst/backends.rst
@@ -341,6 +341,14 @@ can be an arbitrary prefix that will be prepended to all object names
used by S3QL. This allows you to store several S3QL file systems in
the same bucket.
+`s3c://` authenticates API requests using AWS V2 signatures, which are
+deprecated by AWS but still accepted by many S3 compatible services.
+
+`s3c4://` denotes a variant of this backend that works the same
+but uses AWS V4 signatures for request authentication instead: ::
+
+ s3c4://<hostname>:<port>/<bucketname>/<prefix>
+
The S3 compatible backend accepts the following backend options:
.. option:: no-ssl
@@ -385,6 +393,13 @@ The S3 compatible backend accepts the following backend options:
necessary if your storage server does not return a valid response
body for a successful copy operation.
+.. option:: sig-region=<region>
+
+ For `s3c4://` variant only: Region to use for calculating V4
+ request signatures. Contrary to S3, the region is not a defined
+ part of the storage URL and must be specified separately.
+ Defaults to `us-east-1`.
+
.. _`S3 COPY API`: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
.. __: https://doc.s3.amazonaws.com/proposals/copy.html
diff --git a/src/s3ql/backends/__init__.py b/src/s3ql/backends/__init__.py
index a1335762..442828cd 100644
--- a/src/s3ql/backends/__init__.py
+++ b/src/s3ql/backends/__init__.py
@@ -6,7 +6,7 @@
This work can be distributed under the terms of the GNU GPLv3.
'''
-from . import gs, local, rackspace, s3, s3c, swift, swiftks
+from . import gs, local, rackspace, s3, s3c, s3c4, swift, swiftks
from .b2.b2_backend import B2Backend
#: Mapping from storage URL prefixes to backend classes
@@ -15,6 +15,7 @@
'local': local.Backend,
'gs': gs.Backend,
's3c': s3c.Backend,
+ 's3c4': s3c4.Backend,
'swift': swift.Backend,
'swiftks': swiftks.Backend,
'rackspace': rackspace.Backend,
diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py
index e05a49ba..5548a855 100644
--- a/src/s3ql/backends/s3.py
+++ b/src/s3ql/backends/s3.py
@@ -15,7 +15,7 @@
from xml.sax.saxutils import escape as xml_escape
from ..logging import QuietError
-from . import s3c
+from . import s3c4
from .common import retry
from .s3c import get_S3Error
@@ -28,22 +28,23 @@
# pylint: disable=E1002,E1101
-class Backend(s3c.Backend):
+class Backend(s3c4.Backend):
"""A backend to store data in Amazon S3
This class uses standard HTTP connections to connect to S3.
"""
- known_options = (s3c.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
+ known_options = (s3c4.Backend.known_options | {'sse', 'rrs', 'ia', 'oia', 'it'}) - {
'dumb-copy',
'disable-expect100',
+ 'sig-region',
}
def __init__(self, options):
self.region = None
- self.signing_key = None
super().__init__(options)
self._set_storage_options(self._extra_put_headers)
+ self.sig_region = self.region
def _parse_storage_url(self, storage_url, ssl_context):
hit = re.match(r'^s3s?://([^/]+)/([^/]+)(?:/(.*))?$', storage_url)
@@ -147,94 +148,3 @@ def _delete_multi(self, keys):
except:
self.conn.discard()
-
- def _authorize_request(self, method, path, headers, subres, query_string):
- '''Add authorization information to *headers*'''
-
- # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
-
- now = time.gmtime()
- # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
- # '%a, %d %b %Y %H:%M:%S GMT')
-
- ymd = time.strftime('%Y%m%d', now)
- ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
-
- headers['x-amz-date'] = ymdhms
- headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
- # headers['x-amz-content-sha256'] = hashlib.sha256(body).hexdigest()
- headers.pop('Authorization', None)
-
- auth_strs = [method]
- auth_strs.append(urllib.parse.quote(path))
-
- if query_string:
- s = urllib.parse.urlencode(
- query_string, doseq=True, quote_via=urllib.parse.quote
- ).split('&')
- else:
- s = []
- if subres:
- s.append(urllib.parse.quote(subres) + '=')
- if s:
- s = '&'.join(sorted(s))
- else:
- s = ''
- auth_strs.append(s)
-
- # Headers
- sig_hdrs = sorted(x.lower() for x in headers.keys())
- for hdr in sig_hdrs:
- auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
- auth_strs.append('')
- auth_strs.append(';'.join(sig_hdrs))
- auth_strs.append(headers['x-amz-content-sha256'])
- can_req = '\n'.join(auth_strs)
- # log.debug('canonical request: %s', can_req)
-
- can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
- str_to_sign = (
- "AWS4-HMAC-SHA256\n"
- + ymdhms
- + '\n'
- + '%s/%s/s3/aws4_request\n' % (ymd, self.region)
- + can_req_hash
- )
- # log.debug('string to sign: %s', str_to_sign)
-
- if self.signing_key is None or self.signing_key[1] != ymd:
- self.update_signing_key(ymd)
- signing_key = self.signing_key[0]
-
- sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
-
- cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
- self.login,
- now.tm_year,
- now.tm_mon,
- now.tm_mday,
- self.region,
- )
-
- headers['Authorization'] = (
- 'AWS4-HMAC-SHA256 '
- 'Credential=%s,'
- 'SignedHeaders=%s,'
- 'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
- )
-
- def update_signing_key(self, ymd):
- date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
- region_key = hmac_sha256(date_key, self.region.encode())
- service_key = hmac_sha256(region_key, b's3')
- signing_key = hmac_sha256(service_key, b'aws4_request')
-
- self.signing_key = (signing_key, ymd)
-
-
-def hmac_sha256(key, msg, hex=False):
- d = hmac.new(key, msg, hashlib.sha256)
- if hex:
- return d.hexdigest()
- else:
- return d.digest()
diff --git a/src/s3ql/backends/s3c4.py b/src/s3ql/backends/s3c4.py
new file mode 100644
index 00000000..37ff0b7a
--- /dev/null
+++ b/src/s3ql/backends/s3c4.py
@@ -0,0 +1,140 @@
+'''
+s3c4.py - this file is part of S3QL.
+
+Copyright © 2008 Nikolaus Rath <Nikolaus@rath.org>
+
+This work can be distributed under the terms of the GNU GPLv3.
+'''
+
+import hashlib
+import hmac
+import logging
+import re
+import time
+import urllib.parse
+from xml.sax.saxutils import escape as xml_escape
+
+from ..logging import QuietError
+from . import s3c
+from .common import retry
+from .s3c import get_S3Error
+
+log = logging.getLogger(__name__)
+
+# Maximum number of keys that can be deleted at once
+MAX_KEYS = 1000
+
+# Pylint goes berserk with false positives
+# pylint: disable=E1002,E1101
+
+
+class Backend(s3c.Backend):
+ """A backend to stored data in some S3 compatible storage service.
+
+ This classes uses AWS Signature V4 for authorization.
+ """
+
+ known_options = s3c.Backend.known_options | {'sig-region'}
+
+ def __init__(self, options):
+ self.sig_region = options.backend_options.get('sig-region', 'us-east-1')
+ self.signing_key = None
+ super().__init__(options)
+
+ def __str__(self):
+ return 's3c4://%s/%s/%s' % (self.hostname, self.bucket_name, self.prefix)
+
+ def _authorize_request(self, method, path, headers, subres, query_string):
+ '''Add authorization information to *headers*'''
+
+ # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
+
+ now = time.gmtime()
+ # now = time.strptime('Fri, 24 May 2013 00:00:00 GMT',
+ # '%a, %d %b %Y %H:%M:%S GMT')
+
+ ymd = time.strftime('%Y%m%d', now)
+ ymdhms = time.strftime('%Y%m%dT%H%M%SZ', now)
+
+ # add non-standard port to host header, needed for correct signature
+ if self.port != 443:
+ headers['host'] = '%s:%s' % (self.hostname, self.port)
+
+ headers['x-amz-date'] = ymdhms
+ headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
+
+ headers.pop('Authorization', None)
+
+ auth_strs = [method]
+ auth_strs.append(urllib.parse.quote(path))
+
+ if query_string:
+ s = urllib.parse.urlencode(
+ query_string, doseq=True, quote_via=urllib.parse.quote
+ ).split('&')
+ else:
+ s = []
+ if subres:
+ s.append(urllib.parse.quote(subres) + '=')
+ if s:
+ s = '&'.join(sorted(s))
+ else:
+ s = ''
+ auth_strs.append(s)
+
+ # Headers
+ sig_hdrs = sorted(x.lower() for x in headers.keys())
+ for hdr in sig_hdrs:
+ auth_strs.append('%s:%s' % (hdr, headers[hdr].strip()))
+ auth_strs.append('')
+ auth_strs.append(';'.join(sig_hdrs))
+ auth_strs.append(headers['x-amz-content-sha256'])
+ can_req = '\n'.join(auth_strs)
+ # log.debug('canonical request: %s', can_req)
+
+ can_req_hash = hashlib.sha256(can_req.encode()).hexdigest()
+ str_to_sign = (
+ "AWS4-HMAC-SHA256\n"
+ + ymdhms
+ + '\n'
+ + '%s/%s/s3/aws4_request\n' % (ymd, self.sig_region)
+ + can_req_hash
+ )
+ # log.debug('string to sign: %s', str_to_sign)
+
+ if self.signing_key is None or self.signing_key[1] != ymd:
+ self.update_signing_key(ymd)
+ signing_key = self.signing_key[0]
+
+ sig = hmac_sha256(signing_key, str_to_sign.encode(), hex=True)
+
+ cred = '%s/%04d%02d%02d/%s/s3/aws4_request' % (
+ self.login,
+ now.tm_year,
+ now.tm_mon,
+ now.tm_mday,
+ self.sig_region,
+ )
+
+ headers['Authorization'] = (
+ 'AWS4-HMAC-SHA256 '
+ 'Credential=%s,'
+ 'SignedHeaders=%s,'
+ 'Signature=%s' % (cred, ';'.join(sig_hdrs), sig)
+ )
+
+ def update_signing_key(self, ymd):
+ date_key = hmac_sha256(("AWS4" + self.password).encode(), ymd.encode())
+ region_key = hmac_sha256(date_key, self.sig_region.encode())
+ service_key = hmac_sha256(region_key, b's3')
+ signing_key = hmac_sha256(service_key, b'aws4_request')
+
+ self.signing_key = (signing_key, ymd)
+
+
+def hmac_sha256(key, msg, hex=False):
+ d = hmac.new(key, msg, hashlib.sha256)
+ if hex:
+ return d.hexdigest()
+ else:
+ return d.digest()
diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py
index 272e10c7..24ad50f4 100644
--- a/src/s3ql/parse_args.py
+++ b/src/s3ql/parse_args.py
@@ -374,7 +374,7 @@ def storage_url_type(s):
# slash (even when using a prefix), but we can't do that now because it
# would make file systems created without trailing slash inaccessible.
if re.match(r'^(s3|gs)://[^/]+$', s) or re.match(
- r'^(s3c|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
+ r'^(s3c|s3c4|swift(ks)?|rackspace)://[^/]+/[^/]+$', s
):
s += '/'
diff --git a/tests/mock_server.py b/tests/mock_server.py
index b453e705..e3084065 100644
--- a/tests/mock_server.py
+++ b/tests/mock_server.py
@@ -292,6 +292,16 @@ def send_error(self, status, message=None, code='', resource='', extra_headers=N
self.wfile.write(content)
+class S3C4RequestHandler(S3CRequestHandler):
+ '''Request Handler for s3c4 backend
+
+ Currently identical to S3CRequestHandler since mock request handlers
+ do not check request signatures.
+ '''
+
+ pass
+
+
class BasicSwiftRequestHandler(S3CRequestHandler):
'''A request handler implementing a subset of the OpenStack Swift Interface
@@ -569,6 +579,7 @@ def inline_error(http_status, body):
#: corresponding storage urls
handler_list = [
(S3CRequestHandler, 's3c://%(host)s:%(port)d/s3ql_test'),
+ (S3C4RequestHandler, 's3c4://%(host)s:%(port)d/s3ql_test'),
# Special syntax only for testing against mock server
(BasicSwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),
(CopySwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'),

View file

@ -16,6 +16,13 @@ in with hosts;
"cluster/services/storage/secrets/garage-rpc-secret.age".publicKeys = max ++ map systemKeys [ grail VEGAS prophet ];
"cluster/services/storage/secrets/storage-box-credentials.age".publicKeys = max ++ map systemKeys [ grail VEGAS prophet ];
"secrets/dovecot-ldap-token.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/gitlab-db-credentials.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/gitlab-initial-root-password.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/gitlab-openid-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/gitlab-secret-db.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/gitlab-secret-jws.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/gitlab-secret-otp.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/gitlab-secret-secret.age".publicKeys = max ++ map systemKeys [ VEGAS ];
"secrets/hyprspace-key-checkmate.age".publicKeys = max ++ map systemKeys [ checkmate ];
"secrets/hyprspace-key-grail.age".publicKeys = max ++ map systemKeys [ grail ];
"secrets/hyprspace-key-thunderskin.age".publicKeys = max ++ map systemKeys [ thunderskin ];

Binary file not shown.

View file

@ -0,0 +1,12 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A XRrOSniXZk7nvufR3liJ3ocjX257eenhQUYZdlYCpl4
ctZGdEgc9SgWka/3R/2WW4G9m1DHIk7HLKaBNyUeHtE
-> ssh-ed25519 5/zT0w k3z9vLsjCPABV2kTRMC3xiriW+4BwSdvnk02Xtoi3zk
w43L1pm8VvwxVp6k8NJA73afZtPGfD8eCb2koa2goZQ
-> ssh-ed25519 d3WGuA Bi1l2WS3kL5Y5NoVh7jAja3BG9LXxem801SSR76j52s
fKhRIb+Ug3sW4JI2rczNnh3Frx/EEnbQfhTUGdwLSo8
-> AOy-grease dju$ xL|5Hh q(A
h0bIKBg8yQBMqNR8M9DlA/wZWWFB+sdo4ApLXvTT19Moz3E5Vly8N2XKHrV3ggCE
Vn2a3snrXDrWxqQgfQEfJo7FnydItRcgO7ZDOuNAlnooyk0
--- 9bMYjHMQsJt4fqnmE2ezRzN4AoKIrlRKAqh8pYRw8SQ
øÜ™j‡râ|ˆ>˜º<CB9C>QÌ7¬p²¾ïÐdð¤hëÝÏ Î3œü»€¤ÃÐÿ57´âð˜{ïžZ9á´é éÖ$DU$—0YÙ º3ÐBMÍã‰ü@oáªU¶_ßÁ¡dÅDݶ<C39D>5jq/¿‰…j`6<36>Z‡îi—åAÄÞ&Q¯”œ¬¢Ê¡*Õ•:R%+ ôò<C3B4>É¡ù£Ì

View file

@ -0,0 +1,11 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A ZCflrN3Tm5CiGr6ajyHWUBB/tQqvBuZkwTrJDrd/aV0
ItnkxqiZTCT77SDnG0JgzaQlDL3LZ96V+kzjxjAJx5s
-> ssh-ed25519 5/zT0w WoKnbgmzpR+HuLdXYCOkPfScle7g7U+NGA/YAmyfIhk
pNfp+gOVyTfnXpVDRXuk16RyjlWjDILrO7Gibh7nRmU
-> ssh-ed25519 d3WGuA L5xjtPNva83jZWsu2bCbcgaDNlou5BFVMsFkR8+L+2Q
4+UtIsyOgY0NAuHtdg4lBJwMyZWquRsmRNeQ+YXqeA0
-> hD-grease q%QV%; &/
jl4ZKGU+SBSR0xhJN0yz7sV2uW/+Yhw
--- 1LIvBjAzD1lUotPXuI4cPHSfUsMFbEaGjE/t+KnQcW4
AWeûۨ˯e¤ c[ ÖÌ 3mÁíyÍΈÐñè6½ g{7rd€_Ê7ØWPö©':ð¢uË ùá¨N

Binary file not shown.

Binary file not shown.

View file

@ -0,0 +1,14 @@
age-encryption.org/v1
-> ssh-ed25519 NO562A su6ATd6CDJ/TD/nAPw1K4ZmELBDdNLZI63DsZl0zCF0
J+2ZXXZArtjDDLIaQL6HaEdawHo8tonMdzHf45IQMO4
-> ssh-ed25519 5/zT0w wdKMnoA5/huvtT/jyj1Aixf9nKtkzcyPSs1yoUpxoAk
yGiW4Zg0h4NGkdU0BZiWzC+72CJZK6pJdrSBuZCVGAE
-> ssh-ed25519 d3WGuA p4QVeohmXdTo8v0Wh2pkEoyqMhZhmdrblBpq39ENnVk
7TybdsMNokMu+2q5ESnvdcNwAeWTl/5XGZltzJ7etjI
-> Q-grease KJL\,Pw& c!aOPX
C6DVdLd90RXPgjf22U5Y8OsW9O9rkfE3kY0LGQhmmjCSZ7yHde4bhOAVNeNronxE
xFy8GtD+ZllI4NPUSyl3Y/90//H2fVUb32WA3Ga5WJmksrGXzg
--- yWDk0jbHXLxwE9jWTT85ORZy0Pw20jaRVihmkKfGnKo
@#
Q)F:ÀŽ¤¶GÍû # 󺡤«L…Ê-k{Tëd+˜´8žà܃üäá/è¹-Žaæ…Ë\O*—°!^Réãy÷@Z/o™~I€
œ[ô°¼POÂ'vüše^ø,…?¢»Òo¼¸]1WƒËFòJëÄ™Ññ¨ôBý&y¼ yŸìVv_<E28098> %ûÇ<C3BB>«'

Binary file not shown.

BIN
secrets/keycloak-dbpass.age Normal file

Binary file not shown.