Merge pull request #55 from privatevoid-net/updates-20230108

Updates 20230108
2023-01-08 22:43:44 +00:00 · 2023-01-08 22:43:44 +00:00 · e571a017bb
commit e571a017bb
parent 57943f57b0 b7ab8a7085
22 changed files with 423 additions and 392 deletions
--- a/cluster/services/hercules-ci-multi-agent/common.nix
+++ b/cluster/services/hercules-ci-multi-agent/common.nix
@ -3,7 +3,6 @@
 let
  mapAgents = lib.flip lib.mapAttrs config.services.hercules-ci-agents;

-  #lib.foldl' (a: b: a // b) {} (lib.attrValues (lib.mapAttrs (basename: basevalue: lib.mapAttrs' (n: v: lib.nameValuePair "${n}-${basename}" v) basevalue) x));
  mergeMap = f: let
    outputs = mapAgents f;
  in  lib.pipe outputs [
@ -38,12 +37,6 @@ in
      group = "hci-${name}";
    };
  });
-  services.hercules-ci-agents.private-void = {
-    settings = {
-      clusterJoinTokenPath = config.age.secrets.hci-token-private-void.path;
-      binaryCachesPath = config.age.secrets.hci-cache-config-private-void.path;
-    };
-  };
  systemd.services = mergeMap (name: _: {
    hercules-ci-agent = {
      # hercules-ci-agent-restarter should take care of this
--- a/cluster/services/hercules-ci-multi-agent/orgs/max.nix
+++ b/cluster/services/hercules-ci-multi-agent/orgs/max.nix
@ -1,7 +1,8 @@
-{ config, inputs, pkgs, ... }:
+{ config, lib, inputs, pkgs, ... }:

 {
  services.hercules-ci-agents.max = {
+    package = lib.mkForce inputs.self.packages.${pkgs.system}.hercules-ci-agent;
    settings = {
      clusterJoinTokenPath = config.age.secrets.hci-token-max.path;
      binaryCachesPath = config.age.secrets.hci-cache-config-max.path;
--- a/cluster/services/hercules-ci-multi-agent/orgs/nixpak.nix
+++ b/cluster/services/hercules-ci-multi-agent/orgs/nixpak.nix
@ -1,7 +1,8 @@
-{ config, inputs, pkgs, ... }:
+{ config, lib, inputs, pkgs, ... }:

 {
  services.hercules-ci-agents.nixpak = {
+    package = lib.mkForce inputs.self.packages.${pkgs.system}.hercules-ci-agent;
    settings = {
      clusterJoinTokenPath = config.age.secrets.hci-token-nixpak.path;
      binaryCachesPath = config.age.secrets.hci-cache-config-nixpak.path;
--- a/cluster/services/hercules-ci-multi-agent/orgs/private-void.nix
+++ b/cluster/services/hercules-ci-multi-agent/orgs/private-void.nix
@ -1,7 +1,8 @@
-{ config, inputs, pkgs, ... }:
+{ config, lib, inputs, pkgs, ... }:

 {
  services.hercules-ci-agents.private-void = {
+    package = lib.mkForce inputs.self.packages.${pkgs.system}.hercules-ci-agent;
    settings = {
      clusterJoinTokenPath = config.age.secrets.hci-token-private-void.path;
      binaryCachesPath = config.age.secrets.hci-cache-config-private-void.path;
--- a/flake.lock
+++ b/flake.lock
@ -121,19 +121,17 @@
        "nixpkgs": [
          "nixpkgs"
        ],
-        "poetry2nix": [
-          "poetry2nix"
-        ],
+        "poetry2nix": "poetry2nix",
        "pre-commit-hooks": [
          "blank"
        ]
      },
      "locked": {
-        "lastModified": 1672407173,
-        "narHash": "sha256-lmUyHBOtD2XuXqthfXOo5ofv4+tRJV9LF0W8Urn3e6o=",
+        "lastModified": 1672661134,
+        "narHash": "sha256-WqBUyKeiv+jI11ug+qP0OnZ4nngK6eBRVTGHgdzEGvc=",
        "owner": "nix-community",
        "repo": "dream2nix",
-        "rev": "7ca3383c2ca2b49b5a0990fc898e4f2fa7848e62",
+        "rev": "9f6911c78dcb0832f7fcc955e847db1a5a9ce29a",
        "type": "github"
      },
      "original": {
@ -349,11 +347,11 @@
      },
      "locked": {
        "host": "git.privatevoid.net",
-        "lastModified": 1671896375,
-        "narHash": "sha256-TmPI8DOUy+CBnMo6j7ZZ3LT42eo6l3DvqunpLkiW4Is=",
+        "lastModified": 1673041460,
+        "narHash": "sha256-D1KJzWwgXD6/fh8DSFbLD3v4BSOPOSvjZIjvmC7u2bs=",
        "owner": "max",
        "repo": "nix-super",
-        "rev": "8cc8551afdf9197dc13c0252c4bc07cab8af88e1",
+        "rev": "1c35324a97a725fe261643ca4dee2128f3af79a6",
        "type": "gitlab"
      },
      "original": {
@ -363,39 +361,13 @@
        "type": "gitlab"
      }
    },
-    "nixos-command": {
-      "inputs": {
-        "flake-utils": [
-          "repin-flake-utils"
-        ],
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "poetry2nix": [
-          "poetry2nix"
-        ]
-      },
-      "locked": {
-        "lastModified": 1662057103,
-        "narHash": "sha256-8czeYcJPbKFcX/F/GuBjgIOXzqNwqr0iOjY5mImgSaM=",
-        "owner": "max-privatevoid",
-        "repo": "nixos-command",
-        "rev": "9d1015b4d366fab6ae46f830e1699a7b9b5dea18",
-        "type": "github"
-      },
-      "original": {
-        "owner": "max-privatevoid",
-        "repo": "nixos-command",
-        "type": "github"
-      }
-    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1672437037,
-        "narHash": "sha256-mVMu/vwx0/bLHD7J7xlItIJpauoMi4V5s26Kh/UzX+s=",
+        "lastModified": 1673163619,
+        "narHash": "sha256-B33PFBL64ZgTWgMnhFL3jgheAN/DjHPsZ1Ih3z0VE5I=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "9af32915445545bf1710d419c0a7918da6acb869",
+        "rev": "8c54d842d9544361aac5f5b212ba04e4089e8efe",
        "type": "github"
      },
      "original": {
@ -406,24 +378,18 @@
      }
    },
    "poetry2nix": {
-      "inputs": {
-        "flake-utils": [
-          "repin-flake-utils"
-        ],
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
+      "flake": false,
      "locked": {
-        "lastModified": 1671550109,
-        "narHash": "sha256-4Iixlro1t75ze8TqXvfz0HTZ9TOBav3vMTuwuYqsxRE=",
+        "lastModified": 1666918719,
+        "narHash": "sha256-BkK42fjAku+2WgCOv2/1NrPa754eQPV7gPBmoKQBWlc=",
        "owner": "nix-community",
        "repo": "poetry2nix",
-        "rev": "f18984f99c67654a1f9fbde463c88a319a2c843e",
+        "rev": "289efb187123656a116b915206e66852f038720e",
        "type": "github"
      },
      "original": {
        "owner": "nix-community",
+        "ref": "1.36.0",
        "repo": "poetry2nix",
        "type": "github"
      }
@ -480,9 +446,7 @@
        "nar-serve": "nar-serve",
        "nix-filter": "nix-filter",
        "nix-super": "nix-super",
-        "nixos-command": "nixos-command",
        "nixpkgs": "nixpkgs",
-        "poetry2nix": "poetry2nix",
        "repin-flake-utils": "repin-flake-utils"
      }
    }
--- a/flake.nix
+++ b/flake.nix
@ -139,7 +139,6 @@
        gomod2nix.follows = "blank";
        mach-nix.follows = "blank";
        nix-pypi-fetcher.follows = "blank";
-        poetry2nix.follows = "poetry2nix";
        pre-commit-hooks.follows = "blank";
      };
    };
@ -190,23 +189,6 @@

    blank.url = "github:divnix/blank";

-    poetry2nix = {
-      url = "github:nix-community/poetry2nix";
-      inputs = {
-        nixpkgs.follows = "nixpkgs";
-        flake-utils.follows = "repin-flake-utils";
-      };
-    };
-
-    nixos-command = {
-      url = "github:max-privatevoid/nixos-command";
-      inputs = {
-        flake-utils.follows = "repin-flake-utils";
-        nixpkgs.follows = "nixpkgs";
-        poetry2nix.follows = "poetry2nix";
-      };
-    };
-
    drv-parts = {
      url = "github:DavHau/drv-parts";
      inputs = {
--- a/modules/maintenance/default.nix
+++ b/modules/maintenance/default.nix
@ -1,9 +1,6 @@
 { inputs, pkgs, ... }:

 {
-  environment.systemPackages = [
-    inputs.nixos-command.packages.${pkgs.system}.nixos-command
-  ];
  systemd.targets.maintenance = {
    unitConfig.AllowIsolate = true;
    wants = [
--- a/packages/build-support/default.nix
+++ b/packages/build-support/default.nix
@ -1,11 +1,11 @@
-{ pkgs, ... }:
-
 {
  imports = [
-    ./options.nix
-
    ./drv-parts
  ];
+  perSystem = { pkgs, ... }: {
+    imports = [
+      ./options.nix
+    ];

    builders = rec {
      fetchAsset = pkgs.callPackage ./fetch-asset { };
@ -14,4 +14,5 @@
        inherit fetchAsset;
      };
    };
+  };
 }
--- a/packages/build-support/drv-parts/default.nix
+++ b/packages/build-support/drv-parts/default.nix
@ -1,6 +1,6 @@
-{ config, inputs, ... }:
-
+{ inputs, ... }:
 {
+  perSystem = { config, ... }: {
    imports = [
      ./backends
      ./dependency-sets
@ -8,4 +8,5 @@
    _module.args = {
      drv-backends = inputs.drv-parts.drv-backends // config.drv-backends;
    };
+  };
 }
--- a/packages/checks/default.nix
+++ b/packages/checks/default.nix
@ -1,5 +1,7 @@
-{ filters, pkgs, self, self', ... }:
+{ self, ... }:
+
 {
+  perSystem = { filters, pkgs, self', ... }: {
    checks = filters.doFilter filters.checks {
      keycloak = pkgs.callPackage ./keycloak-custom-jre.nix {
        jre = self'.packages.jre17_standard;
@ -12,4 +14,5 @@
        inherit (self'.packages) tempo;
      };
    };
+  };
 }
--- a/packages/modules/devshell.nix
+++ b/packages/modules/devshell.nix
@ -1,4 +1,7 @@
-{ lib, config, inputs', system, ... }:
+{ lib, ... }:
+
+{
+  perSystem = { config, inputs', system, ... }:
  with lib;
  let
    inherit (inputs'.devshell.legacyPackages) mkShell;
@ -63,5 +66,5 @@ in {
      });
    };
    config.devShells = lib.mkIf (system == "x86_64-linux") (mapAttrs (_: mkProjectShell) config.projectShells);
+  };
 }
-
--- a/packages/networking/hyprspace/project.nix
+++ b/packages/networking/hyprspace/project.nix
@ -1,6 +1,7 @@
-{ inputs, lib, pkgs, ... }:
+{ lib, inputs, ... }:

 {
+  perSystem = { pkgs, ... }: {
    projectShells.hyprspace = {
      tools = [
        pkgs.go_1_18
@ -37,4 +38,5 @@
        platforms = platforms.linux ++ platforms.darwin;
      };
    };
+  };
 }
--- a/packages/networking/ipfs-cluster/project.nix
+++ b/packages/networking/ipfs-cluster/project.nix
@ -1,6 +1,6 @@
-{ inputs, lib, pkgs, ... }:
-
+{ inputs, ... }:
 {
+  perSystem = { lib, pkgs, ... }: {
    projectShells.ipfs-cluster = {
      tools = [
        pkgs.go_1_19
@ -55,4 +55,5 @@
        maintainers = with maintainers; [ Luflosi jglukasik ]; 
      }; 
    };
+  };
 }
--- a/packages/patched-inputs.nix
+++ b/packages/patched-inputs.nix
@ -1,5 +1,5 @@
 {
-  perSystem = { filters, inputs', ... }:
+  perSystem = { filters, inputs', lib, pkgs, ... }:

  let
    tools = import ./lib/tools.nix;
@ -14,7 +14,19 @@

      agenix = packages.agenix.agenix.override { nix = nix-super; };

-      hercules-ci-agent = packages.hercules-ci-agent.hercules-ci-agent;
+      # hci-agent's build code does some funny shenanigans
+      hercules-ci-agent = let
+        original = packages.hercules-ci-agent.hercules-ci-agent;
+        patchedNix = patch-rename-direct original.nix ({ version, ...}: "nix-${version}_hci1") "patches/extra/hercules-ci-agent/nix";
+      in (original.override {
+        # for hercules-ci-cnix-expr, hercules-ci-cnix-store
+        nix = patchedNix;
+        # for cachix
+        pkgs = pkgs // { nix = patchedNix; };
+      }).overrideAttrs (old: {
+        # for hercules-ci-agent
+        buildInputs = (lib.remove original.nix old.buildInputs) ++ [ patchedNix ];
+      });

      hci = packages.hercules-ci-agent.hercules-ci-cli;
    };
--- a/packages/poetry2nix-overrides/default.nix
+++ b/packages/poetry2nix-overrides/default.nix
@ -1,7 +0,0 @@
-final: prev: {
-  py-multibase = prev.py-multibase.overridePythonAttrs (old: {
-    nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
-      final.pytestrunner
-    ];
-  });
-}
--- a/packages/projects.nix
+++ b/packages/projects.nix
@ -1,6 +1,17 @@
 { inputs, self, ... }:

 {
+  imports = [
+    ./checks
+    ./modules/devshell.nix
+    ./build-support
+
+    ./networking/hyprspace/project.nix
+    ./networking/ipfs-cluster/project.nix
+    ./servers/reflex-cache/project.nix
+    ./websites/landing/project.nix
+    ./websites/stop-using-nix-env/project.nix
+  ];
  perSystem = { filters, pkgs, self', ... }:
  let
    inherit (self'.packages) nix-super;
@ -14,25 +25,8 @@
        overridesDirs = [ ./dream2nix-overrides ];
      };
    };
-    poetry2nix = pkgs.poetry2nix.overrideScope' (final: prev: {
-      defaultPoetryOverrides = prev.defaultPoetryOverrides.extend (import ./poetry2nix-overrides);
-    });
-
  in
  {
-    _module.args = { inherit inputs self; };
-
-    imports = [
-      ./checks
-      ./modules/devshell.nix
-      ./build-support
-
-      ./networking/hyprspace/project.nix
-      ./networking/ipfs-cluster/project.nix
-      ./servers/reflex-cache/project.nix
-      ./websites/landing/project.nix
-      ./websites/stop-using-nix-env/project.nix
-    ];
    packages = filters.doFilter filters.packages rec {
      cinny = pkgs.callPackage ./web-apps/cinny { inherit pins; };

--- a/packages/servers/reflex-cache/project.nix
+++ b/packages/servers/reflex-cache/project.nix
@ -1,6 +1,7 @@
-{ config, drv-backends, inputs, lib, pkgs, ... }:
+{ inputs, ... }: 

-let
+{
+  perSystem = { config, drv-backends, lib, pkgs, ... }: let
    deps = with config.dependencySets.python3Packages; [
      poetry-core
      requests-unixsocket
@ -35,4 +36,5 @@ in
        ];
      };
    };
+  };
 }
--- a/packages/sources/sources.json
+++ b/packages/sources/sources.json
@ -22,9 +22,9 @@
        "repo": "excalidraw"
      },
      "branch": "master",
-      "revision": "5fcf6a48452b1805323b31e1a7773eec9193ff9c",
-      "url": "https://github.com/excalidraw/excalidraw/archive/5fcf6a48452b1805323b31e1a7773eec9193ff9c.tar.gz",
-      "hash": "1cbzkcxwyycz2nd7i0pj4w8a7jjx2l0q2kwv2dp5g6vpydlw18iw"
+      "revision": "06b45e0cfc9bc93b3565acee2932d643169fb298",
+      "url": "https://github.com/excalidraw/excalidraw/archive/06b45e0cfc9bc93b3565acee2932d643169fb298.tar.gz",
+      "hash": "1xfh90kyxfvpy5vyym1g9rz62zz1zgawyq14rfcjpjrybvqzz18s"
    },
    "searxng": {
      "type": "Git",
@ -34,9 +34,9 @@
        "repo": "searxng"
      },
      "branch": "master",
-      "revision": "e8f72d705209bee09738ba050aeb75927d663c84",
-      "url": "https://github.com/searxng/searxng/archive/e8f72d705209bee09738ba050aeb75927d663c84.tar.gz",
-      "hash": "05rrwfd88h6n0xpy3idwvlnf7vsw96l5gyb6l2smkbs1rsyyr9nk"
+      "revision": "54e63839db9f99611127c058c0989721b4d3d19b",
+      "url": "https://github.com/searxng/searxng/archive/54e63839db9f99611127c058c0989721b4d3d19b.tar.gz",
+      "hash": "1scmpvrl5a5smw96k14f817cz3n7lssq0i7v8wc3fdmn1fs5vgfm"
    },
    "stevenblack-hosts": {
      "type": "GitRelease",
@ -47,10 +47,10 @@
      },
      "pre_releases": false,
      "version_upper_bound": null,
-      "version": "3.11.45",
-      "revision": "5ad27300a3c2dbd267b4593226b9c35680e2204f",
-      "url": "https://api.github.com/repos/StevenBlack/hosts/tarball/3.11.45",
-      "hash": "0rsflfp1wm8c7h8jpywvrh3gbqivzjjj7d9nakfjsa2ipvfg8clj"
+      "version": "3.11.50",
+      "revision": "0b190200ac4b0fdd788a9334bd6235eb53242b5c",
+      "url": "https://api.github.com/repos/StevenBlack/hosts/tarball/3.11.50",
+      "hash": "1vwc22bg88hbc2dvxhcpaz02md0fhr97vqbf2jwgrp565pnfcflc"
    },
    "tempo": {
      "type": "GitRelease",
@ -75,10 +75,10 @@
      },
      "pre_releases": false,
      "version_upper_bound": null,
-      "version": "1.19.2",
-      "revision": "aec80b53d53454a103490f91ef469e975062a24c",
-      "url": "https://api.github.com/repos/louislam/uptime-kuma/tarball/1.19.2",
-      "hash": "1hhbh9a5w3ilrfmccwbjsc7qpadccc12l0aglh5acnq2gcxkfr69"
+      "version": "1.19.3",
+      "revision": "b8e8c1b9dbb1cd74f6a5ba02c49367ead506370c",
+      "url": "https://api.github.com/repos/louislam/uptime-kuma/tarball/1.19.3",
+      "hash": "19r1npkk73karlbb0vwbyfr31k2rl18ily7rwwvqqjazmgp0j32n"
    }
  },
  "version": 2
--- a/packages/web-apps/searxng/default.nix
+++ b/packages/web-apps/searxng/default.nix
@ -1,18 +1,27 @@
 { lib, nixosTests, python3, python3Packages, npins, pins }:

-with python3Packages;
+let
+  pin = pins.searxng;
+  repo = pin.repository;
+in with python3Packages;

-toPythonModule (buildPythonApplication {
+toPythonModule (buildPythonApplication rec {
  pname = "searxng";
-  version = "1.0.0pre_${builtins.substring 0 7 pins.searxng.revision}";
+  version = "1.0.0pre_${builtins.substring 0 7 pin.revision}";

  src = npins.mkSource pins.searxng;

  postPatch = ''
    sed -i \
      -e 's/==.*$//' \
-      -e 's/fasttext-wheel/fasttext/g' \
+      -e 's/fasttext-predict/fasttext/g' \
      requirements.txt
+    cat >searx/version_frozen.py <<EOF
+    VERSION_STRING="${version}"
+    VERSION_TAG="${pin.revision}"
+    GIT_URL="https://github.com/${repo.owner}/${repo.repo}"
+    GIT_BRANCH="${pin.branch}"
+    EOF
  '';

  preBuild = ''
--- a/packages/websites/landing/project.nix
+++ b/packages/websites/landing/project.nix
@ -1,5 +1,5 @@
-{ builders, lib, pkgs, self', ... }:
-let
+{
+  perSystem = { builders, lib, pkgs, self', ... }: let
    configFile = pkgs.writeText "hugo-config.json" (builtins.toJSON {
      title = "Private Void | Zero-maintenance perfection";
      baseURL = "https://www.privatevoid.net/";
@ -51,4 +51,5 @@ in
        };
      };
    in site;
+  };
 }
--- a/packages/websites/stop-using-nix-env/project.nix
+++ b/packages/websites/stop-using-nix-env/project.nix
@ -1,6 +1,5 @@
-{ pkgs, ... }:
-
 {
+  perSystem = { pkgs, ... }: {
    packages.stop-using-nix-env = let
      site = with pkgs; stdenvNoCC.mkDerivation rec {
        pname = "stop-using-nix-env";
@ -16,4 +15,5 @@
        };
      };
    in site;
+  };
 }
--- a/patches/extra/hercules-ci-agent/nix/GHPR_7489-PDT-TDE-3114-prevent-a-race-condition-when-creating-the-S3-cache.patch
+++ b/patches/extra/hercules-ci-agent/nix/GHPR_7489-PDT-TDE-3114-prevent-a-race-condition-when-creating-the-S3-cache.patch
@ -0,0 +1,70 @@
+diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
+index f4ea739b05d..3e0689534b6 100644
+--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
+@@ -166,16 +166,37 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
+         return i->second;
+     }
+ 
+    std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
+    {
+        auto i = state.caches.find(uri);
+        if (i == state.caches.end()) {
+            auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
+            if (!queryCache.next())
+                return std::nullopt;
+            state.caches.emplace(uri,
+                Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
+        }
+        return getCache(state, uri);
+    }
+
+     void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
+     {
+         retrySQLite<void>([&]() {
+             auto state(_state.lock());
+            SQLiteTxn txn(state->db);
+
+            // To avoid the race, we have to check if maybe someone hasn't yet created
+            // the cache for this URI in the meantime.
+            auto cache(queryCacheRaw(*state, uri));
+ 
+-            // FIXME: race
+            if (cache)
+                return;
+ 
+             state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
+             assert(sqlite3_changes(state->db) == 1);
+             state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+
+            txn.commit();
+         });
+     }
+ 
+@@ -183,21 +204,12 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
+     {
+         return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
+             auto state(_state.lock());
+-
+-            auto i = state->caches.find(uri);
+-            if (i == state->caches.end()) {
+-                auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl));
+-                if (!queryCache.next())
+-                    return std::nullopt;
+-                state->caches.emplace(uri,
+-                    Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
+-            }
+-
+-            auto & cache(getCache(*state, uri));
+-
+            auto cache(queryCacheRaw(*state, uri));
+            if (!cache)
+                return std::nullopt;
+             return CacheInfo {
+-                .wantMassQuery = cache.wantMassQuery,
+-                .priority = cache.priority
+                .wantMassQuery = cache->wantMassQuery,
+                .priority = cache->priority
+             };
+         });
+     }