Merge pull request #10963 from hercules-ci/build-with-nixpkgs-scope

Build with nixpkgs scope

dep-patches/boehmgc-traceable_allocator-public.diff

@@ -1,12 +0,0 @@
-diff --git a/include/gc_allocator.h b/include/gc_allocator.h
-index 597c7f13..587286be 100644
---- a/include/gc_allocator.h
-+++ b/include/gc_allocator.h
-@@ -312,6 +312,7 @@ public:
- 
- template<>
- class traceable_allocator<void> {
-+public:
-   typedef size_t      size_type;
-   typedef ptrdiff_t   difference_type;
-   typedef void*       pointer;

flake.nix

@@ -129,152 +129,20 @@
         {
           nixStable = prev.nix;
 
-          default-busybox-sandbox-shell = final.busybox.override {
+          # A new scope, so that we can use `callPackage` to inject our own interdependencies
-            useMusl = true;
+          # without "polluting" the top level "`pkgs`" attrset.
-            enableStatic = true;
+          # This also has the benefit of providing us with a distinct set of packages
-            enableMinimal = true;
+          # we can iterate over.
-            extraConfig = ''
+          nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix);
-              CONFIG_FEATURE_FANCY_ECHO y
-              CONFIG_FEATURE_SH_MATH y
-              CONFIG_FEATURE_SH_MATH_64 y
 
-              CONFIG_ASH y
+          # The dependencies are in their own scope, so that they don't have to be
-              CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+          # in Nixpkgs top level `pkgs` or `nixComponents`.
-
+          nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix {
-              CONFIG_ASH_ALIAS y
+            inherit inputs stdenv versionSuffix;
-              CONFIG_ASH_BASH_COMPAT y
+            pkgs = final;
-              CONFIG_ASH_CMDCMD y
-              CONFIG_ASH_ECHO y
-              CONFIG_ASH_GETOPTS y
-              CONFIG_ASH_INTERNAL_GLOB y
-              CONFIG_ASH_JOB_CONTROL y
-              CONFIG_ASH_PRINTF y
-              CONFIG_ASH_TEST y
-            '';
-          };
-
-          libgit2-nix = final.libgit2.overrideAttrs (attrs: {
-            src = libgit2;
-            version = libgit2.lastModifiedDate;
-            cmakeFlags = attrs.cmakeFlags or []
-              ++ [ "-DUSE_SSH=exec" ];
           });
 
-          boehmgc-nix = final.boehmgc.override {
+          nix = final.nixComponents.nix;
-            enableLargeConfig = true;
-          };
-
-          libseccomp-nix = final.libseccomp.overrideAttrs (_: rec {
-            version = "2.5.5";
-            src = final.fetchurl {
-              url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
-              hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
-            };
-          });
-
-          # TODO: define everything here instead of top level?
-          nix-components = {
-            inherit (final)
-              nix-util
-              nix-util-test-support
-              nix-util-test
-              nix-util-c
-              nix-store
-              nix-fetchers
-              nix-perl-bindings
-              ;
-          };
-
-          nix-util = final.callPackage ./src/libutil/package.nix {
-            inherit
-              fileset
-              stdenv
-              officialRelease
-              versionSuffix
-              ;
-          };
-
-          nix-util-test-support = final.callPackage ./tests/unit/libutil-support/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
-
-          nix-util-test = final.callPackage ./tests/unit/libutil/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
-
-          nix-util-c = final.callPackage ./src/libutil-c/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
-
-          nix-store = final.callPackage ./src/libstore/package.nix {
-            inherit
-              fileset
-              stdenv
-              officialRelease
-              versionSuffix
-              ;
-            libseccomp = final.libseccomp-nix;
-            busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
-          };
-
-          nix-fetchers = final.callPackage ./src/libfetchers/package.nix {
-            inherit
-              fileset
-              stdenv
-              officialRelease
-              versionSuffix
-              ;
-          };
-
-          nix =
-            final.callPackage ./package.nix {
-              inherit
-                fileset
-                stdenv
-                officialRelease
-                versionSuffix
-                ;
-              boehmgc = final.boehmgc-nix;
-              libgit2 = final.libgit2-nix;
-              libseccomp = final.libseccomp-nix;
-              busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
-            };
-
-          nix-perl-bindings = final.callPackage ./src/perl/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
-
-          nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
-
-          nix-external-api-docs = final.callPackage ./src/external-api-docs/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
 
           nix_noTests = final.nix.override {
             doCheck = false;
@@ -296,7 +164,7 @@
       # 'nix-perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
 
-      hydraJobs = import ./maintainers/hydra.nix {
+      hydraJobs = import ./packaging/hydra.nix {
         inherit
           inputs
           binaryTarball
@@ -340,7 +208,7 @@
           "static-" = nixpkgsFor.${system}.static;
         })
         (nixpkgsPrefix: nixpkgs: 
-          flatMapAttrs nixpkgs.nix-components
+          flatMapAttrs nixpkgs.nixComponents
             (pkgName: pkg:
               flatMapAttrs pkg.tests or {}
               (testName: test: {
@@ -357,8 +225,8 @@
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
           default = self.packages.${system}.nix;
-          nix-internal-api-docs = nixpkgsFor.${system}.native.nix-internal-api-docs;
+          nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
-          nix-external-api-docs = nixpkgsFor.${system}.native.nix-external-api-docs;
+          nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
         }
         # We need to flatten recursive attribute sets of derivations to pass `flake check`.
         // flatMapAttrs
@@ -373,16 +241,16 @@
           }
           (pkgName: {}: {
               # These attributes go right into `packages.<system>`.
-              "${pkgName}" = nixpkgsFor.${system}.native.${pkgName};
+              "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
-              "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName};
+              "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName};
             }
             // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: {
               # These attributes go right into `packages.<system>`.
-              "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName};
+              "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
             })
             // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: {
               # These attributes go right into `packages.<system>`.
-              "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName};
+              "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName};
             })
           )
         // lib.optionalAttrs (builtins.elem system linux64BitSystems) {
@@ -444,19 +312,19 @@
           };
 
           mesonFlags =
-            map (transformFlag "libutil") pkgs.nix-util.mesonFlags
+            map (transformFlag "libutil") pkgs.nixComponents.nix-util.mesonFlags
-            ++ map (transformFlag "libstore") pkgs.nix-store.mesonFlags
+            ++ map (transformFlag "libstore") pkgs.nixComponents.nix-store.mesonFlags
-            ++ map (transformFlag "libfetchers") pkgs.nix-fetchers.mesonFlags
+            ++ map (transformFlag "libfetchers") pkgs.nixComponents.nix-fetchers.mesonFlags
-            ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nix-perl-bindings.mesonFlags)
+            ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nixComponents.nix-perl-bindings.mesonFlags)
             ;
 
           nativeBuildInputs = attrs.nativeBuildInputs or []
-            ++ pkgs.nix-util.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-util.nativeBuildInputs
-            ++ pkgs.nix-store.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-store.nativeBuildInputs
-            ++ pkgs.nix-fetchers.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
-            ++ lib.optionals havePerl pkgs.nix-perl-bindings.nativeBuildInputs
+            ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
-            ++ pkgs.nix-internal-api-docs.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
-            ++ pkgs.nix-external-api-docs.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
             ++ [
               modular.pre-commit.settings.package
               (pkgs.writeScriptBin "pre-commit-hooks-install"

package.nix

@@ -1,12 +1,10 @@
 { lib
-, fetchurl
 , stdenv
 , releaseTools
 , autoconf-archive
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
-, buildPackages
 , nlohmann_json
 , bison
 , boost
@@ -15,7 +13,6 @@
 , curl
 , editline
 , readline
-, fileset
 , flex
 , git
 , gtest
@@ -50,7 +47,6 @@
 , pname ? "nix"
 
 , versionSuffix ? ""
-, officialRelease ? false
 
 # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix
 , doBuild ? true
@@ -113,6 +109,8 @@
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   # selected attributes with defaults, will be used to define some

packaging/components.nix

@@ -0,0 +1,28 @@
+scope:
+let
+  inherit (scope) callPackage;
+in
+
+# This becomes the pkgs.nixComponents attribute set
+{
+  nix = callPackage ../package.nix { };
+
+  nix-util = callPackage ../src/libutil/package.nix { };
+
+  nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { };
+
+  nix-util-test = callPackage ../tests/unit/libutil/package.nix { };
+
+  nix-util-c = callPackage ../src/libutil-c/package.nix { };
+
+  nix-store = callPackage ../src/libstore/package.nix { };
+
+  nix-fetchers = callPackage ../src/libfetchers/package.nix { };
+
+  nix-perl-bindings = callPackage ../src/perl/package.nix { };
+
+  nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { };
+
+  nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { };
+
+}

packaging/dependencies.nix

@@ -0,0 +1,58 @@
+# These overrides are applied to the dependencies of the Nix components.
+
+{
+  # Flake inputs; used for sources
+  inputs,
+
+  # The raw Nixpkgs, not affected by this scope
+  pkgs,
+
+  stdenv,
+  versionSuffix,
+}:
+scope: {
+  inherit stdenv versionSuffix;
+
+  libseccomp = pkgs.libseccomp.overrideAttrs (_: rec {
+    version = "2.5.5";
+    src = pkgs.fetchurl {
+      url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
+      hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
+    };
+  });
+
+  boehmgc = pkgs.boehmgc.override {
+    enableLargeConfig = true;
+  };
+
+  libgit2 = pkgs.libgit2.overrideAttrs (attrs: {
+    src = inputs.libgit2;
+    version = inputs.libgit2.lastModifiedDate;
+    cmakeFlags = attrs.cmakeFlags or []
+      ++ [ "-DUSE_SSH=exec" ];
+  });
+
+  busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
+    useMusl = true;
+    enableStatic = true;
+    enableMinimal = true;
+    extraConfig = ''
+      CONFIG_FEATURE_FANCY_ECHO y
+      CONFIG_FEATURE_SH_MATH y
+      CONFIG_FEATURE_SH_MATH_64 y
+
+      CONFIG_ASH y
+      CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+      CONFIG_ASH_ALIAS y
+      CONFIG_ASH_BASH_COMPAT y
+      CONFIG_ASH_CMDCMD y
+      CONFIG_ASH_ECHO y
+      CONFIG_ASH_GETOPTS y
+      CONFIG_ASH_INTERNAL_GLOB y
+      CONFIG_ASH_JOB_CONTROL y
+      CONFIG_ASH_PRINTF y
+      CONFIG_ASH_TEST y
+    '';
+  });
+}

packaging/hydra.nix

@@ -9,7 +9,6 @@
 }:
 let
   inherit (inputs) nixpkgs nixpkgs-regression;
-  inherit (lib) fileset;
 
   installScriptFor = tarballs:
     nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix {
@@ -25,14 +24,15 @@ let
            lib.versionAtLeast client.version "2.4pre20211005")
           "-${client.version}-against-${daemon.version}";
 
-      inherit fileset;
-
       test-client = client;
       test-daemon = daemon;
 
       doBuild = false;
     };
 
+  # Technically we could just return `pkgs.nixComponents`, but for Hydra it's
+  # convention to transpose it, and to transpose it efficiently, we need to
+  # enumerate them manually, so that we don't evaluate unnecessary package sets.
   forAllPackages = lib.genAttrs [
     "nix"
     "nix-util"
@@ -46,16 +46,16 @@ in
 {
   # Binary package for various platforms.
   build = forAllPackages (pkgName:
-    forAllSystems (system: nixpkgsFor.${system}.native.${pkgName}));
+    forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName}));
 
   shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
 
   buildStatic = forAllPackages (pkgName:
-    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName}));
+    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName}));
 
   buildCross = forAllPackages (pkgName:
     forAllCrossSystems (crossSystem:
-      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName})));
+      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})));
 
   buildNoGc = forAllSystems (system:
     self.packages.${system}.nix.override { enableGC = false; }
@@ -73,7 +73,7 @@ in
   );
 
   # Perl bindings for various platforms.
-  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix-perl-bindings);
+  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings);
 
   # Binary tarball for various platforms, containing a Nix store
   # with the closure of 'nix' package, and the second half of
@@ -122,10 +122,10 @@ in
   };
 
   # API docs for Nix's unstable internal C++ interfaces.
-  internal-api-docs = nixpkgsFor.x86_64-linux.native.nix-internal-api-docs;
+  internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs;
 
   # API docs for Nix's C bindings.
-  external-api-docs = nixpkgsFor.x86_64-linux.native.nix-external-api-docs;
+  external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs;
 
   # System tests.
   tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // {

src/external-api-docs/package.nix

@@ -1,7 +1,5 @@
 { lib
 , stdenv
-, releaseTools
-, fileset
 
 , meson
 , ninja
@@ -12,6 +10,10 @@
 , versionSuffix ? ""
 }:
 
+let
+  inherit (lib) fileset;
+in
+
 stdenv.mkDerivation (finalAttrs: {
   pname = "nix-external-api-docs";
   version = lib.fileContents ./.version + versionSuffix;

src/internal-api-docs/package.nix

@@ -1,7 +1,5 @@
 { lib
 , stdenv
-, releaseTools
-, fileset
 
 , meson
 , ninja
@@ -12,6 +10,10 @@
 , versionSuffix ? ""
 }:
 
+let
+  inherit (lib) fileset;
+in
+
 stdenv.mkDerivation (finalAttrs: {
   pname = "nix-internal-api-docs";
   version = lib.fileContents ./.version + versionSuffix;

src/libfetchers/package.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , releaseTools
-, fileset
 
 , meson
 , ninja
@@ -16,17 +15,16 @@
 # Configuration Options
 
 , versionSuffix ? ""
-, officialRelease ? false
 
 # Check test coverage of Nix. Probably want to use with with at least
 # one of `doCheck` or `doInstallCheck` enabled.
 , withCoverageChecks ? false
 
-# Avoid setting things that would interfere with a functioning devShell
-, forDevShell ? false
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   mkDerivation =

src/libstore/package.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , releaseTools
-, fileset
 
 , meson
 , ninja
@@ -13,7 +12,6 @@
 , aws-sdk-cpp
 , libseccomp
 , nlohmann_json
-, man
 , sqlite
 
 , busybox-sandbox-shell ? null
@@ -21,7 +19,6 @@
 # Configuration Options
 
 , versionSuffix ? ""
-, officialRelease ? false
 
 # Check test coverage of Nix. Probably want to use with at least
 # one of `doCheck` or `doInstallCheck` enabled.
@@ -32,6 +29,8 @@
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   mkDerivation =

src/libutil-c/package.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , releaseTools
-, fileset
 
 , meson
 , ninja
@@ -19,6 +18,8 @@
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   mkDerivation =

src/libutil/package.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , releaseTools
-, fileset
 
 , meson
 , ninja
@@ -18,7 +17,6 @@
 # Configuration Options
 
 , versionSuffix ? ""
-, officialRelease ? false
 
 # Check test coverage of Nix. Probably want to use with at least
 # one of `doCheck` or `doInstallCheck` enabled.
@@ -26,6 +24,8 @@
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   mkDerivation =

src/perl/package.nix

@@ -1,5 +1,4 @@
 { lib
-, fileset
 , stdenv
 , perl
 , perlPackages
@@ -16,6 +15,10 @@
 , versionSuffix ? ""
 }:
 
+let
+  inherit (lib) fileset;
+in
+
 perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: {
   pname = "nix-perl";
   version = lib.fileContents ./.version + versionSuffix;

tests/unit/libutil-support/package.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , releaseTools
-, fileset
 
 , meson
 , ninja
@@ -21,6 +20,8 @@
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   mkDerivation =

tests/unit/libutil/package.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
 , releaseTools
-, fileset
 
 , meson
 , ninja
@@ -25,6 +24,8 @@
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   mkDerivation =