From 39fd4705ac604c9c211b93ef0e6fbfa6d6d3b0d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Nov 2024 14:04:21 -0500 Subject: [PATCH 1/4] Factor out the dev shell It had gotten rather big. Hopefully we'll eventually have some generic infra for a "multi-package dev shell" and not need so much code for this, but until then it's better in a separate file. --- flake.nix | 91 +-------------------------------------- packaging/dev-shell.nix | 94 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 89 deletions(-) create mode 100644 packaging/dev-shell.nix diff --git a/flake.nix b/flake.nix index 303779c2b..df274f205 100644 --- a/flake.nix +++ b/flake.nix @@ -294,95 +294,8 @@ }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: - let - buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - modular = devFlake.getSystem stdenv.buildPlatform.system; - transformFlag = prefix: flag: - assert builtins.isString flag; - let - rest = builtins.substring 2 (builtins.stringLength flag) flag; - in - "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; - ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; - in { - pname = "shell-for-" + attrs.pname; - - # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop - version = lib.fileContents ./.version; - name = attrs.pname; - - installFlags = "sysconfdir=$(out)/etc"; - shellHook = '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - ''; - - # We use this shell with the local checkout, not unpackPhase. - src = null; - - env = { - # Needed for Meson to find Boost. - # https://github.com/NixOS/nixpkgs/issues/86131. - BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; - BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; - # For `make format`, to work without installing pre-commit - _NIX_PRE_COMMIT_HOOKS_CONFIG = - "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; - }; - - mesonFlags = - map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) - ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) - ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) - ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) - ; - - nativeBuildInputs = attrs.nativeBuildInputs or [] - ++ pkgs.nixComponents.nix-util.nativeBuildInputs - ++ pkgs.nixComponents.nix-store.nativeBuildInputs - ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs - ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs - ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs - ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs - ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs - ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs - ++ lib.optional - (!buildCanExecuteHost - # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 - && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) - && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages - && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) - pkgs.buildPackages.mesonEmulatorHook - ++ [ - pkgs.buildPackages.cmake - pkgs.buildPackages.shellcheck - pkgs.buildPackages.changelog-d - modular.pre-commit.settings.package - (pkgs.writeScriptBin "pre-commit-hooks-install" - modular.pre-commit.settings.installationScript) - ] - # TODO: Remove the darwin check once - # https://github.com/NixOS/nixpkgs/pull/291814 is available - ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (lib.hiPrio pkgs.buildPackages.clang-tools); - - buildInputs = attrs.buildInputs or [] - ++ [ - pkgs.gtest - pkgs.rapidcheck - ] - ++ lib.optional havePerl pkgs.perl - ; - }); - in + makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; }; + in forAllSystems (system: let makeShells = prefix: pkgs: diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix new file mode 100644 index 000000000..4d1e49c96 --- /dev/null +++ b/packaging/dev-shell.nix @@ -0,0 +1,94 @@ +{ lib, devFlake }: + +pkgs: stdenv: + +(pkgs.nix.override { forDevShell = true; }).overrideAttrs (attrs: + +let + buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; +in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ../.version; + name = attrs.pname; + + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH + + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + ''; + + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ pkgs.nixComponents.nix-expr.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs + ++ lib.optional + (!buildCanExecuteHost + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] + # TODO: Remove the darwin check once + # https://github.com/NixOS/nixpkgs/pull/291814 is available + ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (lib.hiPrio pkgs.buildPackages.clang-tools); + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; +}) From 9e1bc9c224a4a33fac36ce7e425266405f5afff3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Nov 2024 15:13:10 -0500 Subject: [PATCH 2/4] Make the choice of stdenv for the dev shell properly affect all deps We have per-stdenv package sets, so we should be using them. --- flake.nix | 28 +++++++++++++++------------- packaging/dev-shell.nix | 3 ++- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/flake.nix b/flake.nix index df274f205..cfb9021d5 100644 --- a/flake.nix +++ b/flake.nix @@ -295,21 +295,23 @@ devShells = let makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; }; + prefixAttrs = prefix: lib.mapAttrs' (k: v: lib.nameValuePair "${prefix}-${k}" v); in forAllSystems (system: - let - makeShells = prefix: pkgs: - lib.mapAttrs' - (k: v: lib.nameValuePair "${prefix}-${k}" v) - (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); - in - (makeShells "native" nixpkgsFor.${system}.native) // - (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static) // - (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // - { - default = self.devShells.${system}.native-stdenvPackages; - } + prefixAttrs "native" (forAllStdenvs (stdenvName: makeShell { + pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages"; + })) // + lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) ( + prefixAttrs "static" (forAllStdenvs (stdenvName: makeShell { + pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsStatic; + })) // + prefixAttrs "cross" (forAllCrossSystems (crossSystem: makeShell { + pkgs = nixpkgsFor.${system}.cross.${crossSystem}; + })) + ) // + { + default = self.devShells.${system}.native-stdenvPackages; + } ); }; } diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 4d1e49c96..2ea880721 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -1,10 +1,11 @@ { lib, devFlake }: -pkgs: stdenv: +{ pkgs }: (pkgs.nix.override { forDevShell = true; }).overrideAttrs (attrs: let + stdenv = pkgs.nixDependencies.stdenv; buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; modular = devFlake.getSystem stdenv.buildPlatform.system; transformFlag = prefix: flag: From 26ea9053123dee2b88da0dc401ca4391f145e5eb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Nov 2024 14:33:09 -0500 Subject: [PATCH 3/4] Include more deps for the dev shell When we get rid of the make build system, we would be missing things. Incuding these packages' deps ensure we don't miss things. --- packaging/dev-shell.nix | 9 +++++---- src/libstore-tests/package.nix | 12 ++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 2ea880721..4b2a87632 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -86,10 +86,11 @@ in { ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (lib.hiPrio pkgs.buildPackages.clang-tools); buildInputs = attrs.buildInputs or [] - ++ [ - pkgs.gtest - pkgs.rapidcheck - ] + ++ pkgs.nixComponents.nix-util.buildInputs + ++ pkgs.nixComponents.nix-store.buildInputs + ++ pkgs.nixComponents.nix-fetchers.buildInputs + ++ pkgs.nixComponents.nix-expr.buildInputs + ++ pkgs.nixComponents.nix-store-tests.externalBuildInputs ++ lib.optional havePerl pkgs.perl ; }) diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 5b2fd108b..3704d8c5c 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -38,15 +38,19 @@ mkMesonExecutable (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - buildInputs = [ - nix-store - nix-store-c - nix-store-test-support + # Hack for sake of the dev shell + passthru.externalBuildInputs = [ sqlite rapidcheck gtest ]; + buildInputs = finalAttrs.passthru.externalBuildInputs ++ [ + nix-store + nix-store-c + nix-store-test-support + ]; + preConfigure = # "Inline" .version so it's not a symlink, and includes the suffix. # Do the meson utils, without modification. From c49a0ae8b8502e26649c8632d01202f3f75e3214 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Nov 2024 23:22:43 -0500 Subject: [PATCH 4/4] Slightly tweak `flake.nix` `lib.concatMapAttrs` instead of `lib.mapAttrs'` and `lib.nameValuePair` Co-authored-by: Robert Hensing --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index cfb9021d5..3ef027dd1 100644 --- a/flake.nix +++ b/flake.nix @@ -295,7 +295,7 @@ devShells = let makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; }; - prefixAttrs = prefix: lib.mapAttrs' (k: v: lib.nameValuePair "${prefix}-${k}" v); + prefixAttrs = prefix: lib.concatMapAttrs (k: v: { "${prefix}-${k}" = v; }); in forAllSystems (system: prefixAttrs "native" (forAllStdenvs (stdenvName: makeShell {