Merge branch 'master' of github.com:NixOS/nix into path-in-exec

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,22 @@
+<!--
+
+IMPORTANT
+
+Nix is a non-trivial project, so for your contribution to be successful,
+it really is important to follow the contributing guidelines:
+
+https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md
+
+Even if you've contributed to open source before, take a moment to read it,
+so you understand the process and the expectations.
+
+- what information to include in commit messages
+- proper attribution
+- volunteering contributions effectively
+- how to get help and our review process.
+
+-->
+
 # Motivation
 <!-- Briefly explain what the change is about and why it is desirable. -->
 

.github/workflows/ci.yml

@@ -49,6 +49,7 @@ jobs:
           done
         ) &
     - run: nix --experimental-features 'nix-command flakes' flake check -L
+    - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json
 
   # Steps to test CI automation in your own fork.
   # Cachix:
@@ -145,7 +146,7 @@ jobs:
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
+    - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v15
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
@@ -195,20 +196,6 @@ jobs:
       - uses: DeterminateSystems/magic-nix-cache-action@main
       - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user
 
-  meson_build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
-      # Only meson packages that don't have a tests.run derivation.
-      # Those that have it are already built and tested as part of nix flake check.
-      - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g')
-
   flake_regressions:
     needs: vm_tests
     runs-on: ubuntu-22.04

CONTRIBUTING.md

@@ -52,6 +52,20 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
 
      Link related issues to inform interested parties and future contributors about your change.
      If your pull request closes one or multiple issues, mention that in the description using `Closes: #<number>`, as it will then happen automatically when your change is merged.
+   * Credit original authors when you're reusing or building on their work.
+   * Link to relevant changes in other projects, so that others can understand the full context of the change in the future when you or someone else will change or troubleshoot the code.
+     This is especially important when your change is based on work done in other repositories.
+
+     Example:
+     ```
+     This is based on the work of @user in <url>.
+     This solution took inspiration from <url>.
+
+     Co-authored-by: User Name <user@example.com>
+     ```
+
+     When cherry-picking from a different repository, use the `-x` flag, and then amend the commits to turn the hashes into URLs.
+
    * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request).
    * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes.
 
@@ -83,4 +97,4 @@ For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/dev
 ## Getting help
 
 Whenever you're stuck or do not know how to proceed, you can always ask for help.
-The appropriate channels to do so can be found on the [NixOS Community](https://nixos.org/community/) page.
+We invite you to use our [Matrix room](https://matrix.to/#/#nix-dev:nixos.org) to ask questions.

Makefile

@@ -38,18 +38,6 @@ makefiles += \
 endif
 endif
 
-ifeq ($(ENABLE_UNIT_TESTS), yes)
-makefiles += \
-  tests/unit/libutil/local.mk \
-  tests/unit/libutil-support/local.mk \
-  tests/unit/libstore/local.mk \
-  tests/unit/libstore-support/local.mk \
-  tests/unit/libfetchers/local.mk \
-  tests/unit/libexpr/local.mk \
-  tests/unit/libexpr-support/local.mk \
-  tests/unit/libflake/local.mk
-endif
-
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 ifdef HOST_UNIX
 makefiles += \
@@ -104,13 +92,6 @@ include mk/lib.mk
 # These must be defined after `mk/lib.mk`. Otherwise the first rule
 # incorrectly becomes the default target.
 
-ifneq ($(ENABLE_UNIT_TESTS), yes)
-.PHONY: check
-check:
-	@echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'."
-	@exit 1
-endif
-
 ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 .PHONY: installcheck
 installcheck:

Makefile.config.in

@@ -12,7 +12,6 @@ ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_DOC_GEN = @ENABLE_DOC_GEN@
 ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@
 ENABLE_S3 = @ENABLE_S3@
-ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@

README.md

@@ -15,7 +15,7 @@ Full reference documentation can be found in the [Nix manual](https://nix.dev/re
 
 ## Building and developing
 
-Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/building.html).
+Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html).
 
 ## Contributing
 

configure.ac

@@ -141,18 +141,6 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
   ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
 AC_SUBST(ENABLE_BUILD)
 
-# Building without unit tests is useful for bootstrapping with a smaller footprint
-# or running the tests in a separate derivation. Otherwise, we do compile and
-# run them.
-
-AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]),
-  ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD)
-AC_SUBST(ENABLE_UNIT_TESTS)
-
-AS_IF(
-  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"],
-  [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])])
-
 AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]),
   ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes)
 AC_SUBST(ENABLE_FUNCTIONAL_TESTS)
@@ -358,16 +346,6 @@ if test "$gc" = yes; then
   CFLAGS="$old_CFLAGS"
 fi
 
-AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[
-
-# Look for gtest.
-PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main])
-
-# Look for rapidcheck.
-PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest])
-
-])
-
 # Look for nlohmann/json.
 PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 

doc/manual/custom.css

@@ -12,8 +12,8 @@ h1.menu-title::before {
 }
 
 
-h1.menu-title {
+.menu-bar {
-  padding: 0.5em;
+  padding: 0.5em 0em;
 }
 
 .sidebar .sidebar-scrollbox {

doc/manual/rl-next/build-hook-default.md

@@ -0,0 +1,22 @@
+---
+synopsis: |-
+  The `build-hook` setting's default is less useful when using `libnixstore` as a library
+prs:
+- 11178
+---
+
+*This is an obscure issue that only affects usage of the `libnixstore` library outside of the Nix executable.*
+
+As part the ongoing [rewrite of the build system](https://github.com/NixOS/nix/issues/2503) to use [Meson](https://mesonbuild.com/), we are also switching to packaging individual Nix components separately (and building them in separate derivations).
+This means that when building `libnixstore` we do not know where the Nix binaries will be installed --- `libnixstore` doesn't know about downstream consumers like the Nix binaries at all.
+
+*This is also unrelated to the _`post`_-`build-hook`*, which is often used for pushing to a cache.*
+
+This has a small adverse affect on remote building --- the `build-remote` executable that is specified from the [`build-hook`](@docroot@/command-ref/conf-file.md#conf-build-hook) setting will not be gotten from the (presumed) installation location, but instead looked up on the `PATH`.
+This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work.
+
+Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221).
+There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder.
+The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/) i.e. `Store` subclass. <!-- we normally don't mention classes, but consider that this release note is about a library use case -->
+
+The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above.

doc/manual/src/development/testing.md

@@ -276,14 +276,12 @@ To ensure that characterisation testing doesn't make it harder to intentionally
 We run the functional tests not just in the build, but also in VM tests.
 This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment.
 
-The recommended way to run these tests during development is:
+These can be run with:
 
 ```shell
-nix build .#hydraJobs.tests.functional_user.quickBuild
+nix build .#hydraJobs.tests.functional_user
 ```
 
-The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`.
-
 Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization.
 
 ## Integration tests
@@ -294,8 +292,6 @@ Because these tests are expensive and require more than what the standard github
 
 You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`.
 
-If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`.
-
 ## Installer tests
 
 After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch.

doc/manual/src/language/constructs/lookup-path.md

@@ -4,11 +4,8 @@
 >
 > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>`
 
-A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry.
+A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry in [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath).
-
+The algorithm for lookup path resolution is described in the documentation on [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile).
-The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath).
-
-See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution.
 
 > **Example**
 >

doc/manual/src/language/operators.md

@@ -3,7 +3,7 @@
 | Name                                   | Syntax                                     | Associativity | Precedence |
 |----------------------------------------|--------------------------------------------|---------------|------------|
 | [Attribute selection]                  | *attrset* `.` *attrpath* \[ `or` *expr* \] | none          | 1          |
-| Function application                   | *func* *expr*                              | left          | 2          |
+| [Function application]                 | *func* *expr*                              | left          | 2          |
 | [Arithmetic negation][arithmetic]      | `-` *number*                               | none          | 3          |
 | [Has attribute]                        | *attrset* `?` *attrpath*                   | none          | 4          |
 | List concatenation                     | *list* `++` *list*                         | right         | 5          |
@@ -32,7 +32,7 @@
 [string]: ./types.md#type-string
 [path]: ./types.md#type-path
 [number]: ./types.md#type-float
-[list]: ./types.md#list
+[list]: ./types.md#type-list
 [attribute set]: ./types.md#attribute-set
 
 <!-- TODO(@rhendric, #10970): ^ rationalize number -> int/float -->
@@ -48,6 +48,22 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe
 
 [Attribute selection]: #attribute-selection
 
+## Function application
+
+> **Syntax**
+>
+> *func* *expr*
+
+Apply the callable value *func* to the argument *expr*. Note the absence of any visible operator symbol.
+A callable value is either:
+- a [user-defined function][function]
+- a [built-in][builtins] function
+- an attribute set with a [`__functor` attribute](./syntax.md#attr-__functor)
+
+> **Warning**
+>
+> [List][list] items are also separated by whitespace, which means that function calls in list items must be enclosed by parentheses.
+
 ## Has attribute
 
 > **Syntax**
@@ -215,3 +231,5 @@ Equivalent to `!`*b1* `||` *b2*.
 > ```
 
 [Pipe operator]: #pipe-operators
+[builtins]: ./builtins.md
+[Function application]: #function-application

doc/manual/src/language/syntax.md

@@ -218,7 +218,7 @@ a string), that attribute is simply not added to the set:
 
 This will evaluate to `{}` if `foo` evaluates to `false`.
 
-A set that has a `__functor` attribute whose value is callable (i.e. is
+A set that has a [`__functor`]{#attr-__functor} attribute whose value is callable (i.e. is
 itself a function or a set with a `__functor` attribute whose value is
 callable) can be applied as if it were a function, with the set itself
 passed in first , e.g.,

flake.lock

@@ -80,11 +80,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1721548954,
+        "lastModified": 1723688146,
-        "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=",
+        "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a",
+        "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,12 +26,6 @@
 
       officialRelease = false;
 
-      version = lib.fileContents ./.version + versionSuffix;
-      versionSuffix =
-        if officialRelease
-        then ""
-        else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
-
       linux32BitSystems = [ "i686-linux" ];
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux32BitSystems ++ linux64BitSystems;
@@ -130,22 +124,21 @@
           # without "polluting" the top level "`pkgs`" attrset.
           # This also has the benefit of providing us with a distinct set of packages
           # we can iterate over.
-          nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix);
+          nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix {
+            inherit (final) lib;
+            inherit officialRelease;
+            src = self;
+          });
 
           # The dependencies are in their own scope, so that they don't have to be
           # in Nixpkgs top level `pkgs` or `nixComponents`.
           nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix {
-            inherit inputs stdenv versionSuffix;
+            inherit inputs stdenv;
             pkgs = final;
           });
 
           nix = final.nixComponents.nix;
 
-          nix_noTests = final.nix.override {
-            doInstallCheck = false;
-            doCheck = false;
-          };
-
           # See https://github.com/NixOS/nixpkgs/pull/214409
           # Remove when fixed in this flake's nixpkgs
           pre-commit =
@@ -170,6 +163,7 @@
           linux64BitSystems
           nixpkgsFor
           self
+          officialRelease
           ;
       };
 
@@ -211,6 +205,9 @@
                 "${nixpkgsPrefix}${pkgName}-${testName}" = test;
               })
             )
+          // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) {
+            "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests;
+          }
         )
       // devFlake.checks.${system} or {}
       );
@@ -220,7 +217,7 @@
           # for which we don't apply the full build matrix such as cross or static.
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
-          default = self.packages.${system}.nix;
+          default = self.packages.${system}.nix-ng;
           nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
           nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
         }
@@ -228,22 +225,48 @@
         // flatMapAttrs
           { # Components we'll iterate over in the upcoming lambda
             "nix" = { };
-            # Temporarily disabled because GitHub Actions OOM issues. Once
+            "nix-util" = { };
-            # the old build system is gone and we are back to one build
+            "nix-util-c" = { };
-            # system, we should reenable these.
+            "nix-util-test-support" = { };
-            #"nix-util" = { };
+            "nix-util-tests" = { };
-            #"nix-store" = { };
+
-            #"nix-fetchers" = { };
+            "nix-store" = { };
+            "nix-store-c" = { };
+            "nix-store-test-support" = { };
+            "nix-store-tests" = { };
+
+            "nix-fetchers" = { };
+            "nix-fetchers-tests" = { };
+
+            "nix-expr" = { };
+            "nix-expr-c" = { };
+            "nix-expr-test-support" = { };
+            "nix-expr-tests" = { };
+
+            "nix-flake" = { };
+            "nix-flake-tests" = { };
+
+            "nix-main" = { };
+            "nix-main-c" = { };
+
+            "nix-cmd" = { };
+
+            "nix-cli" = { };
+
+            "nix-functional-tests" = { supportsCross = false; };
+
+            "nix-perl-bindings" = { supportsCross = false; };
+            "nix-ng" = { };
           }
-          (pkgName: {}: {
+          (pkgName: { supportsCross ? true }: {
               # These attributes go right into `packages.<system>`.
               "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
               "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName};
             }
-            // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: {
+            // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: {
               # These attributes go right into `packages.<system>`.
               "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
-            })
+            }))
             // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: {
               # These attributes go right into `packages.<system>`.
               "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName};
@@ -253,10 +276,10 @@
         dockerImage =
           let
             pkgs = nixpkgsFor.${system}.native;
-            image = import ./docker.nix { inherit pkgs; tag = version; };
+            image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; };
           in
           pkgs.runCommand
-            "docker-image-tarball-${version}"
+            "docker-image-tarball-${pkgs.nix.version}"
             { meta.description = "Docker image with Nix for ${system}"; }
             ''
               mkdir -p $out/nix-support
@@ -324,6 +347,7 @@
             ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
             ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
             ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs
             ++ lib.optional
               (!stdenv.buildPlatform.canExecute stdenv.hostPlatform
                  # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479

maintainers/flake-module.nix

@@ -641,19 +641,8 @@
             ''^tests/functional/selfref-gc\.sh$''
             ''^tests/functional/shell\.sh$''
             ''^tests/functional/shell\.shebang\.sh$''
-            ''^tests/functional/signing\.sh$''
             ''^tests/functional/simple\.builder\.sh$''
-            ''^tests/functional/simple\.sh$''
-            ''^tests/functional/ssh-relay\.sh$''
-            ''^tests/functional/store-info\.sh$''
-            ''^tests/functional/structured-attrs\.sh$''
-            ''^tests/functional/substitute-with-invalid-ca\.sh$''
-            ''^tests/functional/suggestions\.sh$''
             ''^tests/functional/supplementary-groups\.sh$''
-            ''^tests/functional/tarball\.sh$''
-            ''^tests/functional/test-infra\.sh$''
-            ''^tests/functional/test-libstoreconsumer\.sh$''
-            ''^tests/functional/timeout\.sh$''
             ''^tests/functional/toString-path\.sh$''
             ''^tests/functional/user-envs-migration\.sh$''
             ''^tests/functional/user-envs-test-case\.sh$''

meson.build

@@ -42,3 +42,4 @@ subproject('nix-fetchers-tests')
 subproject('nix-expr-test-support')
 subproject('nix-expr-tests')
 subproject('nix-flake-tests')
+subproject('nix-functional-tests')

package.nix

@@ -47,15 +47,12 @@
 
 , pname ? "nix"
 
-, versionSuffix ? ""
+, version
+, versionSuffix
 
 # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix
 , doBuild ? true
 
-# Run the unit tests as part of the build. See `installUnitTests` for an
-# alternative to this.
-, doCheck ? __forDefaults.canRunInstalled
-
 # Run the functional tests as part of the build.
 , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled
 
@@ -88,11 +85,6 @@
 # - readline
 , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
 
-# Whether to install unit tests. This is useful when cross compiling
-# since we cannot run them natively during the build, but can do so
-# later.
-, installUnitTests ? doBuild && !__forDefaults.canExecuteHost
-
 # For running the functional tests against a pre-built Nix. Probably
 # want to use in conjunction with `doBuild = false;`.
 , test-daemon ? null
@@ -112,13 +104,11 @@
 let
   inherit (lib) fileset;
 
-  version = lib.fileContents ./.version + versionSuffix;
-
   # selected attributes with defaults, will be used to define some
   # things which should instead be gotten via `finalAttrs` in order to
   # work with overriding.
   attrs = {
-    inherit doBuild doCheck doInstallCheck;
+    inherit doBuild doInstallCheck;
   };
 
   mkDerivation =
@@ -134,16 +124,11 @@ in
 mkDerivation (finalAttrs: let
 
   inherit (finalAttrs)
-    doCheck
     doInstallCheck
     ;
 
   doBuild = !finalAttrs.dontBuild;
 
-  # Either running the unit tests during the build, or installing them
-  # to be run later, requiresthe unit tests to be built.
-  buildUnitTests = doCheck || installUnitTests;
-
 in {
   inherit pname version;
 
@@ -177,8 +162,6 @@ in {
           ./scripts/local.mk
         ] ++ lib.optionals enableManual [
           ./doc/manual
-        ] ++ lib.optionals buildUnitTests [
-          ./tests/unit
         ] ++ lib.optionals doInstallCheck [
           ./tests/functional
         ]));
@@ -191,8 +174,6 @@ in {
     # If we are doing just build or just docs, the one thing will use
     # "out". We only need additional outputs if we are doing both.
     ++ lib.optional (doBuild && enableManual) "doc"
-    ++ lib.optional installUnitTests "check"
-    ++ lib.optional doCheck "testresults"
     ;
 
   nativeBuildInputs = [
@@ -231,9 +212,6 @@ in {
     ({ inherit readline editline; }.${readlineFlavor})
   ] ++ lib.optionals enableMarkdown [
     lowdown
-  ] ++ lib.optionals buildUnitTests [
-    gtest
-    rapidcheck
   ] ++ lib.optional stdenv.isLinux libseccomp
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
@@ -248,22 +226,16 @@ in {
   );
 
   dontBuild = !attrs.doBuild;
-  doCheck = attrs.doCheck;
 
   configureFlags = [
     (lib.enableFeature doBuild "build")
-    (lib.enableFeature buildUnitTests "unit-tests")
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature enableGC "gc")
     (lib.enableFeature enableMarkdown "markdown")
-    (lib.enableFeature installUnitTests "install-unit-tests")
     (lib.withFeatureAs true "readline-flavor" readlineFlavor)
   ] ++ lib.optionals (!forDevShell) [
     "--sysconfdir=/etc"
-  ] ++ lib.optionals installUnitTests [
-    "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
-    "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
   ] ++ lib.optionals (doBuild) [
     "--with-boost=${boost}/lib"
   ] ++ lib.optionals (doBuild && stdenv.isLinux) [
@@ -344,10 +316,6 @@ in {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
     mainProgram = "nix";
     broken = !(lib.all (a: a) [
-      # We cannot run or install unit tests if we don't build them or
-      # Nix proper (which they depend on).
-      (installUnitTests -> doBuild)
-      (doCheck -> doBuild)
       # The build process for the manual currently requires extracting
       # data from the Nix executable we are trying to document.
       (enableManual -> doBuild)

packaging/components.nix

@@ -1,11 +1,34 @@
+{
+  lib,
+  src,
+  officialRelease,
+}:
+
 scope:
+
 let
   inherit (scope) callPackage;
+
+  baseVersion = lib.fileContents ../.version;
+
+  versionSuffix = lib.optionalString (!officialRelease) "pre";
+
+  fineVersionSuffix = lib.optionalString
+    (!officialRelease)
+    "pre${builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")}_${src.shortRev or "dirty"}";
+
+  fineVersion = baseVersion + fineVersionSuffix;
 in
 
 # This becomes the pkgs.nixComponents attribute set
 {
-  nix = callPackage ../package.nix { };
+  version = baseVersion + versionSuffix;
+  inherit versionSuffix;
+
+  nix = callPackage ../package.nix {
+    version = fineVersion;
+    versionSuffix = fineVersionSuffix;
+  };
 
   nix-util = callPackage ../src/libutil/package.nix { };
   nix-util-c = callPackage ../src/libutil-c/package.nix { };
@@ -33,11 +56,15 @@ in
 
   nix-cmd = callPackage ../src/libcmd/package.nix { };
 
-  # Will replace `nix` once the old build system is gone.
+  nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; };
-  nix-ng = callPackage ../src/nix/package.nix { };
 
-  nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { };
+  nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; };
-  nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { };
+
+  nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; };
+  nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; };
 
   nix-perl-bindings = callPackage ../src/perl/package.nix { };
+
+  # Will replace `nix` once the old build system is gone.
+  nix-ng = callPackage ../packaging/everything.nix { };
 }

packaging/dependencies.nix

@@ -8,7 +8,6 @@
   pkgs,
 
   stdenv,
-  versionSuffix,
 }:
 
 let
@@ -73,11 +72,9 @@ let
       strictDeps = prevAttrs.strictDeps or true;
       enableParallelBuilding = true;
     };
-
 in
 scope: {
-  inherit stdenv versionSuffix;
+  inherit stdenv;
-  version = lib.fileContents ../.version + versionSuffix;
 
   aws-sdk-cpp = (pkgs.aws-sdk-cpp.override {
     apis = [ "s3" "transfer" ];

packaging/everything.nix

@@ -0,0 +1,93 @@
+{
+  lib,
+  stdenv,
+  buildEnv,
+
+  nix-util,
+  nix-util-c,
+  nix-util-test-support,
+  nix-util-tests,
+
+  nix-store,
+  nix-store-c,
+  nix-store-test-support,
+  nix-store-tests,
+
+  nix-fetchers,
+  nix-fetchers-tests,
+
+  nix-expr,
+  nix-expr-c,
+  nix-expr-test-support,
+  nix-expr-tests,
+
+  nix-flake,
+  nix-flake-tests,
+
+  nix-main,
+  nix-main-c,
+
+  nix-cmd,
+
+  nix-cli,
+
+  nix-functional-tests,
+
+  nix-internal-api-docs,
+  nix-external-api-docs,
+
+  nix-perl-bindings,
+}:
+
+(buildEnv rec {
+  name = "nix-${nix-cli.version}";
+  paths = [
+    nix-util
+    nix-util-c
+    nix-util-test-support
+    nix-util-tests
+
+    nix-store
+    nix-store-c
+    nix-store-test-support
+    nix-store-tests
+
+    nix-fetchers
+    nix-fetchers-tests
+
+    nix-expr
+    nix-expr-c
+    nix-expr-test-support
+    nix-expr-tests
+
+    nix-flake
+    nix-flake-tests
+
+    nix-main
+    nix-main-c
+
+    nix-cmd
+
+    nix-cli
+
+    nix-internal-api-docs
+    nix-external-api-docs
+
+  ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [
+    nix-perl-bindings
+  ];
+}).overrideAttrs (_: {
+  doCheck = true;
+  doInstallCheck = true;
+
+  checkInputs = [
+    # Actually run the unit tests too
+    nix-util-tests.tests.run
+    nix-store-tests.tests.run
+    nix-expr-tests.tests.run
+    nix-flake-tests.tests.run
+  ];
+  installCheckInputs = [
+    nix-functional-tests
+  ];
+})

packaging/hydra.nix

@@ -6,6 +6,7 @@
 , linux64BitSystems
 , nixpkgsFor
 , self
+, officialRelease
 }:
 let
   inherit (inputs) nixpkgs nixpkgs-regression;
@@ -16,7 +17,7 @@ let
     };
 
   testNixVersions = pkgs: client: daemon:
-    pkgs.callPackage ../package.nix {
+    pkgs.nixComponents.callPackage ../package.nix {
       pname =
         "nix-tests"
         + lib.optionalString
@@ -28,6 +29,12 @@ let
       test-daemon = daemon;
 
       doBuild = false;
+
+      # This could be more accurate, but a shorter version will match the
+      # fine version with rev. This functionality is already covered in
+      # the normal test, so it's fine.
+      version = pkgs.nixComponents.version;
+      versionSuffix = pkgs.nixComponents.versionSuffix;
     };
 
   # Technically we could just return `pkgs.nixComponents`, but for Hydra it's
@@ -54,6 +61,8 @@ let
     "nix-main"
     "nix-main-c"
     "nix-cmd"
+    "nix-cli"
+    "nix-functional-tests"
     "nix-ng"
   ];
 in
@@ -68,14 +77,16 @@ in
     lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName}));
 
   buildCross = forAllPackages (pkgName:
-    forAllCrossSystems (crossSystem:
+    # Hack to avoid non-evaling package
-      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})));
+    (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id)
+    (forAllCrossSystems (crossSystem:
+      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))));
 
   buildNoGc = forAllSystems (system:
     self.packages.${system}.nix.override { enableGC = false; }
   );
 
-  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests);
+  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli);
 
   # Toggles some settings for better coverage. Windows needs these
   # library combinations, and Debian build Nix with GNU readline too.

src/libcmd/common-eval-args.cc

@@ -171,7 +171,9 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto accessor = fetchers::downloadTarball(
-            EvalSettings::resolvePseudoUrl(s)).accessor;
+            state.store,
+            state.fetchSettings,
+            EvalSettings::resolvePseudoUrl(s));
         auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }

src/libcmd/repl.cc

@@ -7,7 +7,6 @@
 
 #include "ansicolor.hh"
 #include "shared.hh"
-#include "config-global.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
 #include "attr-path.hh"
@@ -77,10 +76,14 @@ struct NixRepl
     int displ;
     StringSet varNames;
 
+    RunNix * runNixPtr;
+
+    void runNix(Path program, const Strings & args, const std::optional<std::string> & input = {});
+
     std::unique_ptr<ReplInteracter> interacter;
 
     NixRepl(const LookupPath & lookupPath, nix::ref<Store> store,ref<EvalState> state,
-            std::function<AnnotatedValues()> getValues);
+            std::function<AnnotatedValues()> getValues, RunNix * runNix);
     virtual ~NixRepl() = default;
 
     ReplExitStatus mainLoop() override;
@@ -125,32 +128,16 @@ std::string removeWhitespace(std::string s)
 
 
 NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
-            std::function<NixRepl::AnnotatedValues()> getValues)
+            std::function<NixRepl::AnnotatedValues()> getValues, RunNix * runNix = nullptr)
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
     , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
+    , runNixPtr{runNix}
     , interacter(make_unique<ReadlineLikeInteracter>(getDataDir() + "/nix/repl-history"))
 {
 }
 
-void runNix(Path program, const Strings & args,
-    const std::optional<std::string> & input = {})
-{
-    auto subprocessEnv = getEnv();
-    subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
-    //isInteractive avoid grabling interactive commands
-    runProgram2(RunOptions {
-        .program = settings.nixBinDir+ "/" + program,
-        .args = args,
-        .environment = subprocessEnv,
-        .input = input,
-        .isInteractive = true,
-    });
-
-    return;
-}
-
 static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt)
 {
     if (dt.isError)
@@ -833,9 +820,18 @@ void NixRepl::evalString(std::string s, Value & v)
 }
 
 
+void NixRepl::runNix(Path program, const Strings & args, const std::optional<std::string> & input)
+{
+    if (runNixPtr)
+        (*runNixPtr)(program, args, input);
+    else
+        throw Error("Cannot run '%s', no method of calling the Nix CLI provided", program);
+}
+
+
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
    const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
-   std::function<AnnotatedValues()> getValues)
+   std::function<AnnotatedValues()> getValues, RunNix * runNix)
 {
     return std::make_unique<NixRepl>(
         lookupPath,

src/libcmd/repl.hh

@@ -19,9 +19,19 @@ struct AbstractNixRepl
 
     typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
 
+    using RunNix = void(Path program, const Strings & args, const std::optional<std::string> & input);
+
+    /**
+     * @param runNix Function to run the nix CLI to support various
+     * `:<something>` commands. Optional; if not provided,
+     * everything else will still work fine, but those commands won't.
+     */
     static std::unique_ptr<AbstractNixRepl> create(
-        const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
+        const LookupPath & lookupPath,
-        std::function<AnnotatedValues()> getValues);
+        nix::ref<Store> store,
+        ref<EvalState> state,
+        std::function<AnnotatedValues()> getValues,
+        RunNix * runNix = nullptr);
 
     static ReplExitStatus runSimple(
         ref<EvalState> evalState,

src/libexpr/eval.cc

@@ -3088,7 +3088,9 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
     if (EvalSettings::isPseudoUrl(value)) {
         try {
             auto accessor = fetchers::downloadTarball(
-                EvalSettings::resolvePseudoUrl(value)).accessor;
+                store,
+                fetchSettings,
+                EvalSettings::resolvePseudoUrl(value));
             auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
             return finish(store->toRealPath(storePath));
         } catch (Error & e) {

src/libexpr/primops.cc

@@ -4857,7 +4857,10 @@ void EvalState::createBaseEnv()
     addConstant("__nixPath", v, {
         .type = nList,
         .doc = R"(
-          The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
+          A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
+          Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are
+          - Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option
+          - Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`
 
           > **Example**
           >

src/libexpr/primops/fetchTree.cc

@@ -507,7 +507,11 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     //       https://github.com/NixOS/nix/issues/4313
     auto storePath =
         unpack
-        ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
+        ? fetchToStore(
+            *state.store,
+            fetchers::downloadTarball(state.store, state.fetchSettings, *url),
+            FetchMode::Copy,
+            name)
         : fetchers::downloadFile(state.store, *url, name).storePath;
 
     if (expectedHash) {

src/libexpr/value.hh

@@ -331,9 +331,9 @@ public:
 
     void mkStringMove(const char * s, const NixStringContext & context);
 
-    inline void mkString(const Symbol & s)
+    inline void mkString(const SymbolStr & s)
     {
-        mkString(((const std::string &) s).c_str());
+        mkString(s.c_str());
     }
 
     void mkPath(const SourcePath & path);

src/libfetchers/tarball.cc

@@ -102,7 +102,7 @@ DownloadFileResult downloadFile(
     };
 }
 
-DownloadTarballResult downloadTarball(
+static DownloadTarballResult downloadTarball_(
     const std::string & url,
     const Headers & headers)
 {
@@ -202,6 +202,22 @@ DownloadTarballResult downloadTarball(
     return attrsToResult(infoAttrs);
 }
 
+ref<SourceAccessor> downloadTarball(
+    ref<Store> store,
+    const Settings & settings,
+    const std::string & url)
+{
+    /* Go through Input::getAccessor() to ensure that the resulting
+       accessor has a fingerprint. */
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "tarball");
+    attrs.insert_or_assign("url", url);
+
+    auto input = Input::fromAttrs(settings, std::move(attrs));
+
+    return input.getAccessor(store).first;
+}
+
 // An input scheme corresponding to a curl-downloadable resource.
 struct CurlInputScheme : InputScheme
 {
@@ -353,7 +369,7 @@ struct TarballInputScheme : CurlInputScheme
     {
         auto input(_input);
 
-        auto result = downloadTarball(getStrAttr(input.attrs, "url"), {});
+        auto result = downloadTarball_(getStrAttr(input.attrs, "url"), {});
 
         result.accessor->setPathDisplay("«" + input.to_string() + "»");
 

src/libfetchers/tarball.hh

@@ -14,6 +14,8 @@ struct SourceAccessor;
 
 namespace nix::fetchers {
 
+struct Settings;
+
 struct DownloadFileResult
 {
     StorePath storePath;
@@ -40,8 +42,9 @@ struct DownloadTarballResult
  * Download and import a tarball into the Git cache. The result is the
  * Git tree hash of the root directory.
  */
-DownloadTarballResult downloadTarball(
+ref<SourceAccessor> downloadTarball(
-    const std::string & url,
+    ref<Store> store,
-    const Headers & headers = {});
+    const Settings & settings,
+    const std::string & url);
 
 }

src/libstore/globals.cc

@@ -64,7 +64,6 @@ Settings::Settings()
     , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
     , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
     , nixUserConfFiles(getUserConfigFiles())
-    , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
@@ -95,34 +94,6 @@ Settings::Settings()
     sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib");
     allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
 #endif
-
-    /* Set the build hook location
-
-       For builds we perform a self-invocation, so Nix has to be self-aware.
-       That is, it has to know where it is installed. We don't think it's sentient.
-
-       Normally, nix is installed according to `nixBinDir`, which is set at compile time,
-       but can be overridden. This makes for a great default that works even if this
-       code is linked as a library into some other program whose main is not aware
-       that it might need to be a build remote hook.
-
-       However, it may not have been installed at all. For example, if it's a static build,
-       there's a good chance that it has been moved out of its installation directory.
-       That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the
-       current executable, using `getSelfExe()`.
-
-       As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible.
-       If you're porting Nix to a new platform, that might be good enough for a while, but
-       you'll want to improve `getSelfExe()` to work on your platform.
-     */
-    std::string nixExePath = nixBinDir + "/nix";
-    if (!pathExists(nixExePath)) {
-        nixExePath = getSelfExe().value_or("nix");
-    }
-    buildHook = {
-        nixExePath,
-        "__build-remote",
-    };
 }
 
 void loadConfFile(AbstractConfig & config)

src/libstore/globals.hh

@@ -84,11 +84,6 @@ public:
      */
     std::vector<Path> nixUserConfFiles;
 
-    /**
-     * The directory where the main programs are stored.
-     */
-    Path nixBinDir;
-
     /**
      * The directory where the man pages are stored.
      */
@@ -246,7 +241,7 @@ public:
         )",
         {"build-timeout"}};
 
-    Setting<Strings> buildHook{this, {}, "build-hook",
+    Setting<Strings> buildHook{this, {"nix", "__build-remote"}, "build-hook",
         R"(
           The path to the helper program that executes remote builds.
 

src/libstore/local.mk

@@ -71,7 +71,6 @@ libstore_CXXFLAGS += \
  -DNIX_STATE_DIR=\"$(NIX_ROOT)$(localstatedir)/nix\" \
  -DNIX_LOG_DIR=\"$(NIX_ROOT)$(localstatedir)/log/nix\" \
  -DNIX_CONF_DIR=\"$(NIX_ROOT)$(sysconfdir)/nix\" \
- -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" \
  -DNIX_MAN_DIR=\"$(NIX_ROOT)$(mandir)\" \
  -DLSOF=\"$(NIX_ROOT)$(lsof)\"
 

src/libstore/meson.build

@@ -21,7 +21,7 @@ configdata = configuration_data()
 # TODO rename, because it will conflict with downstream projects
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 
-configdata.set_quoted('SYSTEM', host_machine.system())
+configdata.set_quoted('SYSTEM', host_machine.cpu_family() + '-' + host_machine.system())
 
 deps_private_maybe_subproject = [
 ]
@@ -328,7 +328,6 @@ prefix = get_option('prefix')
 path_opts = [
   # Meson built-ins.
   'datadir',
-  'bindir',
   'mandir',
   'libdir',
   'includedir',
@@ -373,7 +372,6 @@ cpp_str_defines = {
   'NIX_STATE_DIR': state_dir / 'nix',
   'NIX_LOG_DIR':   log_dir,
   'NIX_CONF_DIR':  sysconfdir / 'nix',
-  'NIX_BIN_DIR':   bindir,
   'NIX_MAN_DIR':   mandir,
 }
 

src/libstore/unix/build/hook-instance.cc

@@ -4,6 +4,7 @@
 #include "file-system.hh"
 #include "child.hh"
 #include "strings.hh"
+#include "executable-path.hh"
 
 namespace nix {
 
@@ -16,11 +17,18 @@ HookInstance::HookInstance()
     if (buildHookArgs.empty())
         throw Error("'build-hook' setting is empty");
 
-    auto buildHook = canonPath(buildHookArgs.front());
+    std::filesystem::path buildHook = buildHookArgs.front();
     buildHookArgs.pop_front();
 
+    try {
+        buildHook = ExecutablePath::load().findPath(buildHook);
+    } catch (ExecutableLookupError & e) {
+        e.addTrace(nullptr, "while resolving the 'build-hook' setting'");
+        throw;
+    }
+
     Strings args;
-    args.push_back(std::string(baseNameOf(buildHook)));
+    args.push_back(buildHook.filename().string());
 
     for (auto & arg : buildHookArgs)
         args.push_back(arg);
@@ -59,7 +67,7 @@ HookInstance::HookInstance()
         if (dup2(builderOut.readSide.get(), 5) == -1)
             throw SysError("dupping builder's stdout/stderr");
 
-        execv(buildHook.c_str(), stringsToCharPtrs(args).data());
+        execv(buildHook.native().c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("executing '%s'", buildHook);
     });

src/libstore/unix/build/sandbox-defaults.sb

@@ -49,6 +49,7 @@ R""(
 (if (param "_ALLOW_LOCAL_NETWORKING")
     (begin
       (allow network* (remote ip "localhost:*"))
+      (allow network-inbound (local ip "*:*")) ; required to bind and listen
 
       ; Allow access to /etc/resolv.conf (which is a symlink to
       ; /private/var/run/resolv.conf).

src/libutil/executable-path.cc

@@ -60,7 +60,7 @@ OsString ExecutablePath::render() const
 }
 
 std::optional<fs::path>
-ExecutablePath::find(const OsString & exe, std::function<bool(const fs::path &)> isExecutable) const
+ExecutablePath::findName(const OsString & exe, std::function<bool(const fs::path &)> isExecutable) const
 {
     // "If the pathname being sought contains a <slash>, the search
     // through the path prefixes shall not be performed."
@@ -76,4 +76,20 @@ ExecutablePath::find(const OsString & exe, std::function<bool(const fs::path &)>
     return std::nullopt;
 }
 
+fs::path ExecutablePath::findPath(const fs::path & exe, std::function<bool(const fs::path &)> isExecutable) const
+{
+    // "If the pathname being sought contains a <slash>, the search
+    // through the path prefixes shall not be performed."
+    // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03
+    if (exe.filename() == exe) {
+        auto resOpt = findName(exe, isExecutable);
+        if (resOpt)
+            return *resOpt;
+        else
+            throw ExecutableLookupError("Could not find executable '%s'", exe.native());
+    } else {
+        return exe;
+    }
+}
+
 } // namespace nix

src/libutil/executable-path.hh

@@ -5,6 +5,8 @@
 
 namespace nix {
 
+MakeError(ExecutableLookupError, Error);
+
 struct ExecutablePath
 {
     std::vector<std::filesystem::path> directories;
@@ -54,10 +56,21 @@ struct ExecutablePath
      *
      * @return path to a resolved executable
      */
-    std::optional<std::filesystem::path> find(
+    std::optional<std::filesystem::path> findName(
         const OsString & exe,
         std::function<bool(const std::filesystem::path &)> isExecutableFile = isExecutableFileAmbient) const;
 
+    /**
+     * Like the `findName` but also allows a file path as input.
+     *
+     * This implements the full POSIX spec: if the path is just a name,
+     * it searches like the above. Otherwise, it returns the path as is.
+     * If (in the name case) the search fails, an exception is thrown.
+     */
+    std::filesystem::path findPath(
+        const std::filesystem::path & exe,
+        std::function<bool(const std::filesystem::path &)> isExecutable = isExecutableFileAmbient) const;
+
     bool operator==(const ExecutablePath &) const = default;
 };
 

src/nix-channel/nix-channel.cc

@@ -7,6 +7,7 @@
 #include "eval-settings.hh" // for defexpr
 #include "users.hh"
 #include "tarball.hh"
+#include "self-exe.hh"
 
 #include <fcntl.h>
 #include <regex>
@@ -67,7 +68,7 @@ static void removeChannel(const std::string & name)
     channels.erase(name);
     writeChannels();
 
-    runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name });
+    runProgram(getNixBin("nix-env").string(), true, { "--profile", profile, "--uninstall", name });
 }
 
 static Path nixDefExpr;
@@ -118,7 +119,7 @@ static void update(const StringSet & channelNames)
 
             bool unpacked = false;
             if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
-                runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
+                runProgram(getNixBin("nix-build").string(), false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
                             "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
                 unpacked = true;
             }
@@ -143,7 +144,7 @@ static void update(const StringSet & channelNames)
     for (auto & expr : exprs)
         envArgs.push_back(std::move(expr));
     envArgs.push_back("--quiet");
-    runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
+    runProgram(getNixBin("nix-env").string(), false, envArgs);
 
     // Make the channels appear in nix-env.
     struct stat st;
@@ -244,7 +245,7 @@ static int main_nix_channel(int argc, char ** argv)
             case cListGenerations:
                 if (!args.empty())
                     throw UsageError("'--list-generations' expects no arguments");
-                std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush;
+                std::cout << runProgram(getNixBin("nix-env").string(), false, {"--profile", profile, "--list-generations"}) << std::flush;
                 break;
             case cRollback:
                 if (args.size() > 1)
@@ -256,7 +257,7 @@ static int main_nix_channel(int argc, char ** argv)
                 } else {
                     envArgs.push_back("--rollback");
                 }
-                runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
+                runProgram(getNixBin("nix-env").string(), false, envArgs);
                 break;
         }
 

src/nix-functional-tests

@@ -0,0 +1 @@
+../tests/functional

src/nix/flake.cc

@@ -438,14 +438,39 @@ struct CmdFlakeCheck : FlakeCommand
 
         auto checkApp = [&](const std::string & attrPath, Value & v, const PosIdx pos) {
             try {
-                #if 0
+                Activity act(*logger, lvlInfo, actUnknown, fmt("checking app '%s'", attrPath));
-                // FIXME
+                state->forceAttrs(v, pos, "");
-                auto app = App(*state, v);
+                if (auto attr = v.attrs()->get(state->symbols.create("type")))
-                for (auto & i : app.context) {
+                    state->forceStringNoCtx(*attr->value, attr->pos, "");
-                    auto [drvPathS, outputName] = NixStringContextElem::parse(i);
+                else
-                    store->parseStorePath(drvPathS);
+                    throw Error("app '%s' lacks attribute 'type'", attrPath);
+
+                if (auto attr = v.attrs()->get(state->symbols.create("program"))) {
+                    if (attr->name == state->symbols.create("program")) {
+                        NixStringContext context;
+                        state->forceString(*attr->value, context, attr->pos, "");
+                    }
+                } else
+                    throw Error("app '%s' lacks attribute 'program'", attrPath);
+
+                if (auto attr = v.attrs()->get(state->symbols.create("meta"))) {
+                    state->forceAttrs(*attr->value, attr->pos, "");
+                    if (auto dAttr = attr->value->attrs()->get(state->symbols.create("description")))
+                        state->forceStringNoCtx(*dAttr->value, dAttr->pos, "");
+                    else
+                        logWarning({
+                            .msg = HintFmt("app '%s' lacks attribute 'meta.description'", attrPath),
+                        });
+                } else
+                    logWarning({
+                        .msg = HintFmt("app '%s' lacks attribute 'meta'", attrPath),
+                    });
+
+                for (auto & attr : *v.attrs()) {
+                    std::string_view name(state->symbols[attr.name]);
+                    if (name != "type" && name != "program" && name != "meta")
+                        throw Error("app '%s' has unsupported attribute '%s'", attrPath, name);
                 }
-                #endif
             } catch (Error & e) {
                 e.addTrace(resolve(pos), HintFmt("while checking the app definition '%s'", attrPath));
                 reportError(e);
@@ -630,7 +655,7 @@ struct CmdFlakeCheck : FlakeCommand
                                 const auto & attr_name = state->symbols[attr.name];
                                 checkSystemName(attr_name, attr.pos);
                                 if (checkSystemType(attr_name, attr.pos)) {
-                                    checkApp(
+                                    checkDerivation(
                                         fmt("%s.%s", name, attr_name),
                                         *attr.value, attr.pos);
                                 };
@@ -1252,8 +1277,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
                         }
                         j.emplace("type", "derivation");
                         j.emplace("name", name);
-                        if (description)
+                        j.emplace("description", description ? *description : "");
-                            j.emplace("description", *description);
                     } else {
                         logger->cout("%s: %s '%s'",
                             headerPrefix,
@@ -1341,12 +1365,19 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
                     (attrPath.size() == 3 && attrPathS[0] == "apps"))
                 {
                     auto aType = visitor.maybeGetAttr("type");
+                    std::optional<std::string> description;
+                    if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) {
+                        if (auto aDescription = aMeta->maybeGetAttr(state->sDescription))
+                            description = aDescription->getString();
+                    }
                     if (!aType || aType->getString() != "app")
                         state->error<EvalError>("not an app definition").debugThrow();
                     if (json) {
                         j.emplace("type", "app");
+                        if (description)
+                            j.emplace("description", *description);
                     } else {
-                        logger->cout("%s: app", headerPrefix);
+                        logger->cout("%s: app: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description ? *description : "no description");
                     }
                 }
 

src/nix/local.mk

@@ -26,6 +26,8 @@ endif
 
 nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix)
 
+nix_CXXFLAGS += -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\"
+
 nix_LIBS = libexpr libmain libfetchers libflake libstore libutil libcmd
 
 nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)

src/nix/main.cc

@@ -19,6 +19,7 @@
 #include "network-proxy.hh"
 #include "eval-cache.hh"
 #include "flake/flake.hh"
+#include "self-exe.hh"
 
 #include <sys/types.h>
 #include <regex>
@@ -366,6 +367,17 @@ void mainWrapped(int argc, char * * argv)
     initGC();
     flake::initLib(flakeSettings);
 
+    /* Set the build hook location
+
+       For builds we perform a self-invocation, so Nix has to be
+       self-aware. That is, it has to know where it is installed. We
+       don't think it's sentient.
+     */
+    settings.buildHook.setDefault(Strings {
+        getNixBin({}).string(),
+        "__build-remote",
+    });
+
     #if __linux__
     if (isRootUser()) {
         try {

src/nix/meson.build

@@ -33,6 +33,21 @@ subdir('build-utils-meson/threads')
 
 subdir('build-utils-meson/export-all-symbols')
 
+configdata = configuration_data()
+
+fs = import('fs')
+
+bindir = get_option('bindir')
+if not fs.is_absolute(bindir)
+  bindir = get_option('prefix') / bindir
+endif
+configdata.set_quoted('NIX_BIN_DIR', bindir)
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-nix-cli.hh',
+)
+
 add_project_arguments(
   # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
   # It would be nice for our headers to be idempotent instead.
@@ -42,15 +57,17 @@ add_project_arguments(
   #'-include', 'config-fetchers.hh',
   '-include', 'config-main.hh',
   '-include', 'config-cmd.hh',
+  '-include', 'config-nix-cli.hh',
   language : 'cpp',
 )
 
 subdir('build-utils-meson/diagnostics')
 subdir('build-utils-meson/generate-header')
 
-nix_sources = files(
+nix_sources = [config_h] + files(
   'add-to-store.cc',
   'app.cc',
+  'self-exe.cc',
   'build.cc',
   'bundle.cc',
   'cat.cc',
@@ -175,3 +192,55 @@ this_exe = executable(
   link_args: linker_export_flags,
   install : true,
 )
+
+meson.override_find_program('nix', this_exe)
+
+nix_symlinks = [
+  'nix-build',
+  'nix-channel',
+  'nix-collect-garbage',
+  'nix-copy-closure',
+  'nix-daemon',
+  'nix-env',
+  'nix-hash',
+  'nix-instantiate',
+  'nix-prefetch-url',
+  'nix-shell',
+  'nix-store',
+]
+
+foreach linkname : nix_symlinks
+  install_symlink(
+    linkname,
+    # TODO(Qyriad): should these continue to be relative symlinks?
+    pointing_to : 'nix',
+    install_dir : get_option('bindir'),
+    # The 'runtime' tag is what executables default to, which we want to emulate here.
+    install_tag : 'runtime'
+  )
+  t = custom_target(
+    command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
+    output: linkname,
+    # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
+    build_by_default: true
+  )
+  # TODO(Ericson3214): Dosen't yet work
+  #meson.override_find_program(linkname, t)
+endforeach
+
+install_symlink(
+  'build-remote',
+  pointing_to : '..' / '..'/ get_option('bindir') / 'nix',
+  install_dir : get_option('libexecdir') / 'nix',
+  # The 'runtime' tag is what executables default to, which we want to emulate here.
+  install_tag : 'runtime'
+)
+
+custom_target(
+  command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
+  output: 'build-remote',
+  # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
+  build_by_default: true
+)
+# TODO(Ericson3214): Dosen't yet work
+#meson.override_find_program(linkname, t)

src/nix/repl.cc

@@ -1,12 +1,32 @@
 #include "eval.hh"
 #include "eval-settings.hh"
+#include "config-global.hh"
 #include "globals.hh"
 #include "command.hh"
 #include "installable-value.hh"
 #include "repl.hh"
+#include "processes.hh"
+#include "self-exe.hh"
 
 namespace nix {
 
+void runNix(Path program, const Strings & args,
+    const std::optional<std::string> & input = {})
+{
+    auto subprocessEnv = getEnv();
+    subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
+    //isInteractive avoid grabling interactive commands
+    runProgram2(RunOptions {
+        .program = getNixBin(program).string(),
+        .args = args,
+        .environment = subprocessEnv,
+        .input = input,
+        .isInteractive = true,
+    });
+
+    return;
+}
+
 struct CmdRepl : RawInstallablesCommand
 {
     CmdRepl() {
@@ -81,7 +101,8 @@ struct CmdRepl : RawInstallablesCommand
             lookupPath,
             openStore(),
             state,
-            getValues
+            getValues,
+            runNix
         );
         repl->autoArgs = getAutoArgs(*repl->state);
         repl->initEnv();

src/nix/run.md

@@ -80,6 +80,7 @@ An app is specified by a flake output attribute named
 apps.x86_64-linux.blender_2_79 = {
   type = "app";
   program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender";
+  meta.description = "Run Blender, a free and open-source 3D creation suite.";
 };
 ```
 
@@ -90,4 +91,6 @@ The only supported attributes are:
 * `program` (required): The full path of the executable to run. It
   must reside in the Nix store.
 
+* `meta.description` (optional): A description of the app.
+
 )""

src/nix/self-exe.cc

@@ -0,0 +1,38 @@
+#include "current-process.hh"
+#include "file-system.hh"
+#include "globals.hh"
+#include "self-exe.hh"
+
+namespace nix {
+
+namespace fs = std::filesystem;
+
+fs::path getNixBin(std::optional<std::string_view> binaryNameOpt)
+{
+    auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; };
+
+    // If the environment variable is set, use it unconditionally
+    if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR"))
+        return fs::path{*envOpt} / std::string{getBinaryName()};
+
+    // Use some-times avaiable OS tricks to get to the path of this Nix, and try that
+    if (auto selfOpt = getSelfExe()) {
+        fs::path path{*selfOpt};
+        if (binaryNameOpt)
+            path = path.parent_path() / std::string{*binaryNameOpt};
+        if (fs::exists(path))
+            return path;
+    }
+
+    // If `nix` exists at the hardcoded fallback path, use it.
+    {
+        auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()};
+        if (fs::exists(path))
+            return path;
+    }
+
+    // return just the name, hoping the exe is on the `PATH`
+    return getBinaryName();
+}
+
+}

src/nix/self-exe.hh

@@ -0,0 +1,31 @@
+#pragma once
+///@file
+
+#include <filesystem>
+
+namespace nix {
+
+/**
+ * Get a path to the given Nix binary.
+ *
+ * Normally, nix is installed according to `NIX_BIN_DIR`, which is set
+ * at compile time, but can be overridden.
+ *
+ * However, it may not have been installed at all. For example, if it's
+ * a static build, there's a good chance that it has been moved out of
+ * its installation directory. That makes `NIX_BIN_DIR` useless.
+ * Instead, we'll query the OS for the path to the current executable,
+ * using `getSelfExe()`.
+ *
+ * As a last resort, we resort to `PATH`. Hopefully we find a `nix`
+ * there that's compatible. If you're porting Nix to a new platform,
+ * that might be good enough for a while, but you'll want to improve
+ * `getSelfExe()` to work on your platform.
+ *
+ * @param binary_name the exact binary name we're looking up. Might be
+ * `nix-*` instead of `nix` for the legacy CLI commands. Optional to use
+ * current binary name.
+ */
+std::filesystem::path getNixBin(std::optional<std::string_view> binary_name = {});
+
+}

src/nix/upgrade-nix.cc

@@ -9,6 +9,7 @@
 #include "names.hh"
 #include "progress-bar.hh"
 #include "executable-path.hh"
+#include "self-exe.hh"
 
 using namespace nix;
 
@@ -93,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir));
-            runProgram(settings.nixBinDir + "/nix-env", false,
+            runProgram(getNixBin("nix-env").string(), false,
                 {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"});
         }
 
@@ -103,7 +104,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
     /* Return the profile in which Nix is installed. */
     Path getProfileDir(ref<Store> store)
     {
-        auto whereOpt = ExecutablePath::load().find(OS_STR("nix-env"));
+        auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env"));
         if (!whereOpt)
             throw Error("couldn't figure out how Nix is installed, so I can't upgrade it");
         auto & where = *whereOpt;

src/perl/lib/Nix/Config.pm.in

@@ -5,7 +5,6 @@ use Nix::Store;
 
 $version = "@PACKAGE_VERSION@";
 
-$binDir = Nix::Store::getBinDir;
 $storeDir = Nix::Store::getStoreDir;
 
 %config = ();

src/perl/lib/Nix/Store.pm

@@ -24,7 +24,7 @@ our @EXPORT = qw(
 
     hashPath hashFile hashString convertHash
     signString checkSignature
-    getBinDir getStoreDir
+    getStoreDir
     setVerbosity
 );
 

src/perl/lib/Nix/Store.xs

@@ -424,11 +424,6 @@ StoreWrapper::addTempRoot(char * storePath)
         }
 
 
-SV * getBinDir()
-    PPCODE:
-        XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0)));
-
-
 SV * getStoreDir()
     PPCODE:
         XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0)));

tests/functional/.version

@@ -0,0 +1 @@
+../../.version

tests/functional/ca/meson.build

@@ -0,0 +1,33 @@
+configure_file(
+  input : 'config.nix.in',
+  output : 'config.nix',
+  configuration : test_confdata,
+)
+
+suites += {
+  'name': 'ca',
+  'deps': [],
+  'tests': [
+    'build-with-garbage-path.sh',
+    'build.sh',
+    'build-cache.sh',
+    'concurrent-builds.sh',
+    'derivation-json.sh',
+    'duplicate-realisation-in-closure.sh',
+    'eval-store.sh',
+    'gc.sh',
+    'import-derivation.sh',
+    'new-build-cmd.sh',
+    'nix-copy.sh',
+    'nix-run.sh',
+    'nix-shell.sh',
+    'post-hook.sh',
+    'recursive.sh',
+    'repl.sh',
+    'selfref-gc.sh',
+    'signatures.sh',
+    'substitute.sh',
+    'why-depends.sh',
+  ],
+  'workdir': meson.current_build_dir(),
+}

tests/functional/common.sh

@@ -8,7 +8,8 @@ COMMON_SH_SOURCED=1
 
 functionalTestsDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")"
 
-source "$functionalTestsDir/common/vars-and-functions.sh"
+source "$functionalTestsDir/common/vars.sh"
+source "$functionalTestsDir/common/functions.sh"
 source "$functionalTestsDir/common/init.sh"
 
 if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then

tests/functional/common/functions.sh

@@ -1,10 +1,10 @@
-# NOTE: instances of @variable@ are substituted as defined in /mk/templates.mk
+# shellcheck shell=bash
 
 set -eu -o pipefail
 
-if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then
+if [[ -z "${COMMON_FUNCTIONS_SH_SOURCED-}" ]]; then
 
-COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1
+COMMON_FUNCTIONS_SH_SOURCED=1
 
 isTestOnNixOS() {
   [[ "${isTestOnNixOS:-}" == 1 ]]
@@ -15,64 +15,14 @@ die() {
   exit 1
 }
 
-set +x
-
-commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")"
-
-source "$commonDir/subst-vars.sh"
-# Make sure shellcheck knows all these will be defined by the above generated snippet
-: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}"
-
-source "$commonDir/paths.sh"
-source "$commonDir/test-root.sh"
-
-test_nix_conf_dir=$TEST_ROOT/etc
-test_nix_conf=$test_nix_conf_dir/nix.conf
-
-export TEST_HOME=$TEST_ROOT/test-home
-
-if ! isTestOnNixOS; then
-  export NIX_STORE_DIR
-  if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
-      # Maybe the build directory is symlinked.
-      export NIX_IGNORE_SYMLINK_STORE=1
-      NIX_STORE_DIR=$TEST_ROOT/store
-  fi
-  export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
-  export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
-  export NIX_STATE_DIR=$TEST_ROOT/var/nix
-  export NIX_CONF_DIR=$test_nix_conf_dir
-  export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
-  unset NIX_USER_CONF_FILES
-  export _NIX_TEST_SHARED=$TEST_ROOT/shared
-  if [[ -n $NIX_STORE ]]; then
-      export _NIX_TEST_NO_SANDBOX=1
-  fi
-  export _NIX_IN_TEST=$TEST_ROOT/shared
-  export _NIX_TEST_NO_LSOF=1
-  export NIX_REMOTE=${NIX_REMOTE_-}
-
-fi # ! isTestOnNixOS
-
-unset NIX_PATH
-export HOME=$TEST_HOME
-unset XDG_STATE_HOME
-unset XDG_DATA_HOME
-unset XDG_CONFIG_HOME
-unset XDG_CONFIG_DIRS
-unset XDG_CACHE_HOME
-
-export IMPURE_VAR1=foo
-export IMPURE_VAR2=bar
-
-cacheDir=$TEST_ROOT/binary-cache
-
 readLink() {
+    # TODO fix this
+    # shellcheck disable=SC2012
     ls -l "$1" | sed 's/.*->\ //'
 }
 
 clearProfiles() {
-    profiles="$HOME"/.local/state/nix/profiles
+    profiles="$HOME/.local/state/nix/profiles"
     rm -rf "$profiles"
 }
 
@@ -105,11 +55,11 @@ doClearStore() {
 }
 
 clearCache() {
-    rm -rf "$cacheDir"
+    rm -rf "${cacheDir?}"
 }
 
 clearCacheCache() {
-    rm -f $TEST_HOME/.cache/nix/binary-cache*
+    rm -f "$TEST_HOME/.cache/nix/binary-cache"*
 }
 
 startDaemon() {
@@ -122,7 +72,7 @@ startDaemon() {
         return
     fi
     # Start the daemon, wait for the socket to appear.
-    rm -f $NIX_DAEMON_SOCKET_PATH
+    rm -f "$NIX_DAEMON_SOCKET_PATH"
     PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon &
     _NIX_TEST_DAEMON_PID=$!
     export _NIX_TEST_DAEMON_PID
@@ -151,14 +101,14 @@ killDaemon() {
     if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then
         return
     fi
-    kill $_NIX_TEST_DAEMON_PID
+    kill "$_NIX_TEST_DAEMON_PID"
     for i in {0..100}; do
-        kill -0 $_NIX_TEST_DAEMON_PID 2> /dev/null || break
+        kill -0 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || break
         sleep 0.1
     done
-    kill -9 $_NIX_TEST_DAEMON_PID 2> /dev/null || true
+    kill -9 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || true
-    wait $_NIX_TEST_DAEMON_PID || true
+    wait "$_NIX_TEST_DAEMON_PID" || true
-    rm -f $NIX_DAEMON_SOCKET_PATH
+    rm -f "$NIX_DAEMON_SOCKET_PATH"
     # Indicate daemon is stopped
     unset _NIX_TEST_DAEMON_PID
     # Restore old nix remote
@@ -177,14 +127,11 @@ restartDaemon() {
     startDaemon
 }
 
-if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
-    _canUseSandbox=1
-fi
-
 isDaemonNewer () {
   [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
   local requiredVersion="$1"
-  local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3)
+  local daemonVersion
+  daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3)
   [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]]
 }
 
@@ -237,7 +184,7 @@ expect() {
     shift
     "$@" && res=0 || res="$?"
     # also match "negative" codes, which wrap around to >127
-    if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then
+    if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then
         echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2
         return 1
     fi
@@ -252,7 +199,7 @@ expectStderr() {
     shift
     "$@" 2>&1 && res=0 || res="$?"
     # also match "negative" codes, which wrap around to >127
-    if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then
+    if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then
         echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2
         return 1
     fi
@@ -267,7 +214,7 @@ expectStderr() {
 #   error: This error is expected
 #   EOF
 assertStderr() {
-    diff -u /dev/stdin <($@ 2>/dev/null 2>&1)
+    diff -u /dev/stdin <("$@" 2>/dev/null 2>&1)
 }
 
 needLocalStore() {
@@ -283,11 +230,9 @@ buggyNeedLocalStore() {
 
 enableFeatures() {
     local features="$1"
-    sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf
+    sed -i 's/experimental-features .*/& '"$features"'/' "${test_nix_conf?}"
 }
 
-set -x
-
 onError() {
     set +x
     echo "$0: test failed at:" >&2
@@ -311,15 +256,15 @@ callerPrefix() {
   local i file line fn savedFn
   # Use `caller`
   for i in $(seq 0 100); do
-    caller $i > /dev/null || {
+    caller "$i" > /dev/null || {
       if [[ -n "${file:-}" ]]; then
         echo "$file:$line: ${savedFn+in call to $savedFn: }"
       fi
       break
     }
-    line="$(caller $i | cut -d' ' -f1)"
+    line="$(caller "$i" | cut -d' ' -f1)"
-    fn="$(caller $i | cut -d' ' -f2)"
+    fn="$(caller "$i" | cut -d' ' -f2)"
-    file="$(caller $i | cut -d' ' -f3)"
+    file="$(caller "$i" | cut -d' ' -f3)"
     if [[ $file != "${BASH_SOURCE[0]}" ]]; then
       echo "$file:$line: ${savedFn+in call to $savedFn: }"
       return
@@ -342,7 +287,7 @@ checkGrepArgs() {
     for arg in "$@"; do
         if [[ "$arg" != "${arg//$'\n'/_}" ]]; then
             echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2
-            return -101
+            return 155 # = -101 mod 256
         fi
     done
 }
@@ -400,4 +345,4 @@ count() {
 
 trap onError ERR
 
-fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED
+fi # COMMON_FUNCTIONS_SH_SOURCED

tests/functional/common/init.sh

@@ -7,10 +7,10 @@ if isTestOnNixOS; then
 
   mkdir -p "$test_nix_conf_dir" "$TEST_HOME"
 
-  export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf"
+  export NIX_USER_CONF_FILES="$test_nix_conf"
   mkdir -p "$test_nix_conf_dir" "$TEST_HOME"
   ! test -e "$test_nix_conf"
-  cat > "$test_nix_conf_dir/nix.conf" <<EOF
+  cat > "$test_nix_conf" <<EOF
 # TODO: this is not needed for all tests and prevents stable commands from be tested in isolation
 experimental-features = nix-command flakes
 flake-registry = $TEST_ROOT/registry.json

tests/functional/common/meson.build

@@ -0,0 +1,5 @@
+configure_file(
+  input : 'subst-vars.sh.in',
+  output : 'subst-vars.sh',
+  configuration : test_confdata,
+)

tests/functional/common/paths.sh

@@ -1,14 +1,25 @@
 # shellcheck shell=bash
 
+set -eu -o pipefail
+
+if [[ -z "${COMMON_PATHS_SH_SOURCED-}" ]]; then
+
+COMMON_PATHS_SH_SOURCED=1
+
 commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")"
 
-# Since this is a generated file
+# Since these are generated files
+# shellcheck disable=SC1091
+source "$commonDir/functions.sh"
 # shellcheck disable=SC1091
 source "$commonDir/subst-vars.sh"
 # Make sure shellcheck knows this will be defined by the above generated snippet
-: "${bindir?}"
+: "${bash?}" "${bindir?}"
 
-export PATH="$bindir:$PATH"
+if ! isTestOnNixOS; then
+  export SHELL="$bash"
+  export PATH="$bindir:$PATH"
+fi
 
 if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then
   export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH
@@ -18,3 +29,5 @@ DAEMON_PATH="$PATH"
 if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
   DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH"
 fi
+
+fi # COMMON_PATHS_SH_SOURCED

tests/functional/common/subst-vars.sh.in

@@ -4,22 +4,14 @@ if [[ -z "${COMMON_SUBST_VARS_SH_SOURCED-}" ]]; then
 
 COMMON_SUBST_VARS_SH_SOURCED=1
 
+bash=@bash@
 bindir=@bindir@
-export coreutils=@coreutils@
+coreutils=@coreutils@
-#lsof=@lsof@
 
-export dot=@dot@
+dot=@dot@
-export PAGER=cat
+busybox="@sandbox_shell@"
-export busybox="@sandbox_shell@"
 
-export version=@PACKAGE_VERSION@
+version=@PACKAGE_VERSION@
-export system=@system@
+system=@system@
-
-export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@
-
-if ! isTestOnNixOS; then
-  export SHELL="@bash@"
-  export PATH=@bindir@:$PATH
-fi
 
 fi

tests/functional/common/vars.sh

@@ -0,0 +1,72 @@
+# shellcheck shell=bash
+
+set -eu -o pipefail
+
+if [[ -z "${COMMON_VARS_SH_SOURCED-}" ]]; then
+
+COMMON_VARS_SH_SOURCED=1
+
+commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")"
+
+# Since this is a generated file
+# shellcheck disable=SC1091
+source "$commonDir/subst-vars.sh"
+# Make sure shellcheck knows all these will be defined by the above generated snippet
+: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${busybox?} ${version?} ${system?}"
+export coreutils dot busybox version system
+
+export PAGER=cat
+
+source "$commonDir/paths.sh"
+source "$commonDir/test-root.sh"
+
+test_nix_conf_dir=$TEST_ROOT/etc
+# Used in other files
+# shellcheck disable=SC2034
+test_nix_conf=$test_nix_conf_dir/nix.conf
+
+export TEST_HOME=$TEST_ROOT/test-home
+
+if ! isTestOnNixOS; then
+  export NIX_STORE_DIR
+  if ! NIX_STORE_DIR=$(readlink -f "$TEST_ROOT/store" 2> /dev/null); then
+      # Maybe the build directory is symlinked.
+      export NIX_IGNORE_SYMLINK_STORE=1
+      NIX_STORE_DIR=$TEST_ROOT/store
+  fi
+  export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
+  export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
+  export NIX_STATE_DIR=$TEST_ROOT/var/nix
+  export NIX_CONF_DIR=$test_nix_conf_dir
+  export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
+  unset NIX_USER_CONF_FILES
+  export _NIX_TEST_SHARED=$TEST_ROOT/shared
+  if [[ -n $NIX_STORE ]]; then
+      export _NIX_TEST_NO_SANDBOX=1
+  fi
+  export _NIX_IN_TEST=$TEST_ROOT/shared
+  export _NIX_TEST_NO_LSOF=1
+  export NIX_REMOTE=${NIX_REMOTE_-}
+
+fi # ! isTestOnNixOS
+
+unset NIX_PATH
+export HOME=$TEST_HOME
+unset XDG_STATE_HOME
+unset XDG_DATA_HOME
+unset XDG_CONFIG_HOME
+unset XDG_CONFIG_DIRS
+unset XDG_CACHE_HOME
+
+export IMPURE_VAR1=foo
+export IMPURE_VAR2=bar
+
+# Used in other files
+# shellcheck disable=SC2034
+cacheDir=$TEST_ROOT/binary-cache
+
+if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
+    _canUseSandbox=1
+fi
+
+fi # COMMON_VARS_SH_SOURCED

tests/functional/derivation-advanced-attributes.sh

@@ -3,7 +3,7 @@
 source common/test-root.sh
 source common/paths.sh
 
-set -o pipefail
+set -eu -o pipefail
 
 source characterisation/framework.sh
 

tests/functional/dyn-drv/meson.build

@@ -0,0 +1,19 @@
+configure_file(
+  input : 'config.nix.in',
+  output : 'config.nix',
+  configuration : test_confdata,
+)
+
+suites += {
+  'name': 'dyn-drv',
+  'deps': [],
+  'tests': [
+    'text-hashed-output.sh',
+    'recursive-mod-json.sh',
+    'build-built-drv.sh',
+    'eval-outputOf.sh',
+    'dep-built-drv.sh',
+    'old-daemon-error-hack.sh',
+  ],
+  'workdir': meson.current_build_dir(),
+}

tests/functional/flakes/check.sh

@@ -91,3 +91,47 @@ nix flake check $flakeDir
 checkRes=$(nix flake check --all-systems --keep-going $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true)
 echo "$checkRes" | grepQuiet "packages.system-1.default"
 echo "$checkRes" | grepQuiet "packages.system-2.default"
+
+cat > $flakeDir/flake.nix <<EOF
+{
+  outputs = { self }: {
+    apps.system-1.default = {
+      type = "app";
+      program = "foo";
+    };
+    apps.system-2.default = {
+      type = "app";
+      program = "bar";
+      meta.description = "baz";
+    };
+  };
+}
+EOF
+
+nix flake check --all-systems $flakeDir
+
+cat > $flakeDir/flake.nix <<EOF
+{
+  outputs = { self }: {
+    apps.system-1.default = {
+      type = "app";
+      program = "foo";
+      unknown-attr = "bar";
+    };
+  };
+}
+EOF
+
+checkRes=$(nix flake check --all-systems $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true)
+echo "$checkRes" | grepQuiet "unknown-attr"
+
+cat > $flakeDir/flake.nix <<EOF
+{
+  outputs = { self }: {
+    formatter.system-1 = "foo";
+  };
+}
+EOF
+
+checkRes=$(nix flake check --all-systems $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true)
+echo "$checkRes" | grepQuiet "formatter.system-1"

tests/functional/flakes/meson.build

@@ -0,0 +1,28 @@
+suites += {
+  'name': 'flakes',
+  'deps': [],
+  'tests': [
+    'flakes.sh',
+    'develop.sh',
+    'edit.sh',
+    'run.sh',
+    'mercurial.sh',
+    'circular.sh',
+    'init.sh',
+    'inputs.sh',
+    'follow-paths.sh',
+    'bundle.sh',
+    'check.sh',
+    'unlocked-override.sh',
+    'absolute-paths.sh',
+    'absolute-attr-paths.sh',
+    'build-paths.sh',
+    'flake-in-submodule.sh',
+    'prefetch.sh',
+    'eval-cache.sh',
+    'search-root.sh',
+    'config.sh',
+    'show.sh',
+  ],
+  'workdir': meson.current_build_dir(),
+}

tests/functional/git-hashing/meson.build

@@ -0,0 +1,8 @@
+suites += {
+  'name': 'git-hashing',
+  'deps': [],
+  'tests': [
+    'simple.sh',
+  ],
+  'workdir': meson.current_build_dir(),
+}

tests/functional/local-overlay-store/common.sh

@@ -1,4 +1,5 @@
-source ../common/vars-and-functions.sh
+source ../common/vars.sh
+source ../common/functions.sh
 
 TODO_NixOS
 

tests/functional/local-overlay-store/meson.build

@@ -0,0 +1,18 @@
+suites += {
+  'name': 'local-overlay-store',
+  'deps': [],
+  'tests': [
+    'check-post-init.sh',
+    'redundant-add.sh',
+    'build.sh',
+    'bad-uris.sh',
+    'add-lower.sh',
+    'delete-refs.sh',
+    'delete-duplicate.sh',
+    'gc.sh',
+    'verify.sh',
+    'optimise.sh',
+    'stale-file-handle.sh',
+  ],
+  'workdir': meson.current_build_dir(),
+}

tests/functional/meson.build

@@ -0,0 +1,266 @@
+project('nix-functional-tests', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.3',
+  license : 'LGPL-2.1-or-later',
+)
+
+fs = import('fs')
+
+# Need to combine source and build trees
+run_command(
+  'rsync',
+  '-a',
+  '--copy-unsafe-links',
+  meson.current_source_dir() / '',
+  meson.current_build_dir() / '',
+)
+# This current-source-escaping relative is no good because we don't know
+# where the build directory will be, therefore we fix it up. Once the
+# Make build system is gone, we should think about doing this better.
+scripts_dir = fs.relative_to(
+  meson.current_source_dir() / '..' / '..' / 'scripts',
+  meson.current_build_dir(),
+)
+run_command(
+  'sed',
+  '-i', meson.current_build_dir() / 'bash-profile.sh',
+  '-e', 's^../../scripts^@0@^'.format(scripts_dir),
+)
+
+nix = find_program('nix')
+bash = find_program('bash', native : true)
+busybox = find_program('busybox', native : true, required : false)
+coreutils = find_program('coreutils', native : true)
+dot = find_program('dot', native : true, required : false)
+
+nix_bin_dir = fs.parent(nix.full_path())
+
+test_confdata = {
+  'bindir': nix_bin_dir,
+  'coreutils': fs.parent(coreutils.full_path()),
+  'dot': dot.found() ? dot.full_path() : '',
+  'bash': bash.full_path(),
+  'sandbox_shell': busybox.found() ? busybox.full_path() : '',
+  'PACKAGE_VERSION': meson.project_version(),
+  'system': host_machine.cpu_family() + '-' + host_machine.system(),
+}
+
+# Just configures `common/vars-and-functions.sh.in`.
+# Done as a subdir() so Meson places it under `common` in the build directory as well.
+subdir('common')
+
+config_nix_in = configure_file(
+  input : 'config.nix.in',
+  output : 'config.nix',
+  configuration : test_confdata,
+)
+
+suites = [
+  {
+    'name' : 'main',
+    'deps': [],
+    'tests': [
+      'test-infra.sh',
+      'gc.sh',
+      'nix-collect-garbage-d.sh',
+      'remote-store.sh',
+      'legacy-ssh-store.sh',
+      'lang.sh',
+      'lang-gc.sh',
+      'characterisation-test-infra.sh',
+      'experimental-features.sh',
+      'fetchMercurial.sh',
+      'gc-auto.sh',
+      'user-envs.sh',
+      'user-envs-migration.sh',
+      'binary-cache.sh',
+      'multiple-outputs.sh',
+      'nix-build.sh',
+      'gc-concurrent.sh',
+      'repair.sh',
+      'fixed.sh',
+      'export-graph.sh',
+      'timeout.sh',
+      'fetchGitRefs.sh',
+      'gc-runtime.sh',
+      'tarball.sh',
+      'fetchGit.sh',
+      'fetchurl.sh',
+      'fetchPath.sh',
+      'fetchTree-file.sh',
+      'simple.sh',
+      'referrers.sh',
+      'optimise-store.sh',
+      'substitute-with-invalid-ca.sh',
+      'signing.sh',
+      'hash-convert.sh',
+      'hash-path.sh',
+      'gc-non-blocking.sh',
+      'check.sh',
+      'nix-shell.sh',
+      'check-refs.sh',
+      'build-remote-input-addressed.sh',
+      'secure-drv-outputs.sh',
+      'restricted.sh',
+      'fetchGitSubmodules.sh',
+      'fetchGitVerification.sh',
+      'readfile-context.sh',
+      'nix-channel.sh',
+      'recursive.sh',
+      'dependencies.sh',
+      'check-reqs.sh',
+      'build-remote-content-addressed-fixed.sh',
+      'build-remote-content-addressed-floating.sh',
+      'build-remote-trustless-should-pass-0.sh',
+      'build-remote-trustless-should-pass-1.sh',
+      'build-remote-trustless-should-pass-2.sh',
+      'build-remote-trustless-should-pass-3.sh',
+      'build-remote-trustless-should-fail-0.sh',
+      'build-remote-with-mounted-ssh-ng.sh',
+      'nar-access.sh',
+      'impure-eval.sh',
+      'pure-eval.sh',
+      'eval.sh',
+      'repl.sh',
+      'binary-cache-build-remote.sh',
+      'search.sh',
+      'logging.sh',
+      'export.sh',
+      'config.sh',
+      'add.sh',
+      'chroot-store.sh',
+      'filter-source.sh',
+      'misc.sh',
+      'dump-db.sh',
+      'linux-sandbox.sh',
+      'supplementary-groups.sh',
+      'build-dry.sh',
+      'structured-attrs.sh',
+      'shell.sh',
+      'brotli.sh',
+      'zstd.sh',
+      'compression-levels.sh',
+      'nix-copy-ssh.sh',
+      'nix-copy-ssh-ng.sh',
+      'post-hook.sh',
+      'function-trace.sh',
+      'fmt.sh',
+      'eval-store.sh',
+      'why-depends.sh',
+      'derivation-json.sh',
+      'derivation-advanced-attributes.sh',
+      'import-derivation.sh',
+      'nix_path.sh',
+      'case-hack.sh',
+      'placeholders.sh',
+      'ssh-relay.sh',
+      'build.sh',
+      'build-delete.sh',
+      'output-normalization.sh',
+      'selfref-gc.sh',
+      'db-migration.sh',
+      'bash-profile.sh',
+      'pass-as-file.sh',
+      'nix-profile.sh',
+      'suggestions.sh',
+      'store-info.sh',
+      'fetchClosure.sh',
+      'completions.sh',
+      'impure-derivations.sh',
+      'path-from-hash-part.sh',
+      'path-info.sh',
+      'toString-path.sh',
+      'read-only-store.sh',
+      'nested-sandboxing.sh',
+      'impure-env.sh',
+      'debugger.sh',
+      'extra-sandbox-profile.sh',
+      'help.sh',
+    ],
+    'workdir': meson.current_build_dir(),
+  },
+]
+
+nix_store = dependency('nix-store', required : false)
+if nix_store.found()
+  subdir('test-libstoreconsumer')
+  suites += {
+    'name': 'libstoreconsumer',
+    'deps': [
+      libstoreconsumer_tester,
+    ],
+    'tests': [
+      'test-libstoreconsumer.sh',
+    ],
+    'workdir': meson.current_build_dir(),
+  }
+
+endif
+
+# Plugin tests require shared libraries support.
+nix_expr = dependency('nix-expr', required : false)
+if nix_expr.found() and get_option('default_library') != 'static'
+  subdir('plugins')
+  suites += {
+    'name': 'plugins',
+    'deps': [
+      libplugintest,
+    ],
+    'tests': [
+      'plugins.sh',
+    ],
+    'workdir': meson.current_build_dir(),
+  }
+endif
+
+subdir('ca')
+subdir('dyn-drv')
+subdir('flakes')
+subdir('git-hashing')
+subdir('local-overlay-store')
+
+foreach suite : suites
+  foreach script : suite['tests']
+    workdir = suite['workdir']
+    prefix = fs.relative_to(workdir, meson.project_build_root())
+
+    script = script
+    # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called
+    # `functional-flakes-show`.
+    name = fs.replace_suffix(prefix / script, '')
+
+    test(
+      name,
+      bash,
+      args: [
+        '-x',
+        '-e',
+        '-u',
+        '-o', 'pipefail',
+        script,
+      ],
+      suite : suite['name'],
+      env : {
+        'TEST_NAME': name,
+        'NIX_REMOTE': '',
+        'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ',
+      },
+      # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine
+      # this can easily drive them to failure. give them more time than default of 30sec
+      timeout : 300,
+      # Used for target dependency/ordering tracking, not adding compiler flags or anything.
+      depends : suite['deps'],
+      workdir : workdir,
+      # Won't pass until man pages are generated
+      should_fail : suite['name'] == 'main' and script == 'help.sh'
+    )
+  endforeach
+endforeach

tests/functional/nested-sandboxing/command.sh

@@ -1,3 +1,5 @@
+set -eu -o pipefail
+
 export NIX_BIN_DIR=$(dirname $(type -p nix))
 # TODO Get Nix and its closure more flexibly
 export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)"

tests/functional/nested-sandboxing/runner.nix

@@ -6,7 +6,10 @@ mkDerivation {
   name = "nested-sandboxing";
   busybox = builtins.getEnv "busybox";
   EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX";
-  buildCommand = if altitude == 0 then ''
+  buildCommand = ''
+    set -x
+    set -eu -o pipefail
+  '' + (if altitude == 0 then ''
     echo Deep enough! > $out
   '' else ''
     cp -r ${../common} ./common
@@ -20,5 +23,5 @@ mkDerivation {
     source ./nested-sandboxing/command.sh
 
     runNixBuild ${storeFun} ${toString altitude} >> $out
-  '';
+  '');
 }

tests/functional/package.nix

@@ -0,0 +1,117 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+, rsync
+
+, jq
+, git
+, mercurial
+, util-linux
+
+, nix-store
+, nix-expr
+, nix-cli
+
+, rapidcheck
+, gtest
+, runCommand
+
+, busybox-sandbox-shell ? null
+
+# Configuration Options
+
+, version
+
+# For running the functional tests against a different pre-built Nix.
+, test-daemon ? null
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-functional-tests";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../scripts/nix-profile.sh.in
+    ../../.version
+    ../../tests/functional
+    ./.
+  ];
+
+  # Hack for sake of the dev shell
+  passthru.baseNativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+    rsync
+
+    jq
+    git
+    mercurial
+  ] ++ lib.optionals stdenv.hostPlatform.isLinux [
+    # For various sandboxing tests that needs a statically-linked shell,
+    # etc.
+    busybox-sandbox-shell
+    # For Overlay FS tests need `mount`, `umount`, and `unshare`.
+    # TODO use `unixtools` to be precise over which executables instead?
+    util-linux
+  ];
+
+  nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [
+    nix-cli
+  ];
+
+  buildInputs = [
+    nix-store
+    nix-expr
+  ];
+
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../../.version
+    ''
+    # TEMP hack for Meson before make is gone, where
+    # `src/nix-functional-tests` is during the transition a symlink and
+    # not the actual directory directory.
+    + ''
+      cd $(readlink -e $PWD)
+      echo $PWD | grep tests/functional
+    '';
+
+  mesonCheckFlags = [
+    "--print-errorlogs"
+  ];
+
+  preCheck =
+    # See https://github.com/NixOS/nix/issues/2523
+    # Occurs often in tests since https://github.com/NixOS/nix/pull/9900
+    lib.optionalString stdenv.hostPlatform.isDarwin ''
+      export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+    '';
+
+  doCheck = true;
+
+  installPhase = ''
+    mkdir $out
+  '';
+
+  meta = {
+    platforms = lib.platforms.unix;
+  };
+
+} // lib.optionalAttrs (test-daemon != null) {
+  NIX_DAEMON_PACKAGE = test-daemon;
+})

tests/functional/plugins.sh

@@ -2,10 +2,11 @@
 
 source common.sh
 
-if [[ $BUILD_SHARED_LIBS != 1 ]]; then
+for ext in so dylib; do
-    skipTest "Plugins are not supported"
+    plugin="$PWD/plugins/libplugintest.$ext"
-fi
+    [[ -f "$plugin" ]] && break
+done
 
-res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull)
+res=$(nix --option setting-set true --option plugin-files "$plugin" eval --expr builtins.anotherNull)
 
 [ "$res"x = "nullx" ]

tests/functional/plugins/meson.build

@@ -0,0 +1,16 @@
+libplugintest = shared_module(
+  'plugintest',
+  'plugintest.cc',
+  cpp_args : [
+    # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+    # It would be nice for our headers to be idempotent instead.
+    '-include', 'config-util.hh',
+    '-include', 'config-store.hh',
+    # '-include', 'config-fetchers.hh',
+    '-include', 'config-expr.hh',
+  ],
+  dependencies : [
+    dependency('nix-expr'),
+  ],
+  build_by_default : false,
+)

tests/functional/restricted.sh

@@ -16,9 +16,6 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.n
 (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix')
 nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../..
 
-(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel')
-nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src
-
 expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' | grepQuiet "forbidden in restricted mode"
 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' -I src=.
 

tests/functional/signing.sh

@@ -5,108 +5,108 @@ source common.sh
 clearStoreIfPossible
 clearCache
 
-nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1
+nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT"/sk1 "$TEST_ROOT"/pk1
-pk1=$(cat $TEST_ROOT/pk1)
+pk1=$(cat "$TEST_ROOT"/pk1)
-nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2
+nix-store --generate-binary-cache-key cache2.example.org "$TEST_ROOT"/sk2 "$TEST_ROOT"/pk2
-pk2=$(cat $TEST_ROOT/pk2)
+pk2=$(cat "$TEST_ROOT"/pk2)
 
 # Build a path.
 outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2")
 
 # Verify that the path got signed.
-info=$(nix path-info --json $outPath)
+info=$(nix path-info --json "$outPath")
-echo $info | jq -e '.[] | .ultimate == true'
+echo "$info" | jq -e '.[] | .ultimate == true'
 TODO_NixOS # looks like an actual bug? Following line fails on NixOS:
-echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))'
+echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))'
-echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))'
+echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))'
 
 # Test "nix store verify".
-nix store verify -r $outPath
+nix store verify -r "$outPath"
 
-expect 2 nix store verify -r $outPath --sigs-needed 1
+expect 2 nix store verify -r "$outPath" --sigs-needed 1
 
-nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1
+nix store verify -r "$outPath" --sigs-needed 1 --trusted-public-keys "$pk1"
 
-expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1
+expect 2 nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1"
 
-nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
+nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
 nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
 # Build something unsigned.
 outPath2=$(nix-build simple.nix --no-out-link)
 
-nix store verify -r $outPath
+nix store verify -r "$outPath"
 
 # Verify that the path did not get signed but does have the ultimate bit.
-info=$(nix path-info --json $outPath2)
+info=$(nix path-info --json "$outPath2")
-echo $info | jq -e '.[] | .ultimate == true'
+echo "$info" | jq -e '.[] | .ultimate == true'
-echo $info | jq -e '.[] | .signatures == []'
+echo "$info" | jq -e '.[] | .signatures == []'
 
 # Test "nix store verify".
-nix store verify -r $outPath2
+nix store verify -r "$outPath2"
 
-expect 2 nix store verify -r $outPath2 --sigs-needed 1
+expect 2 nix store verify -r "$outPath2" --sigs-needed 1
 
-expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
+expect 2 nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1"
 
 # Test "nix store sign".
-nix store sign --key-file $TEST_ROOT/sk1 $outPath2
+nix store sign --key-file "$TEST_ROOT"/sk1 "$outPath2"
 
-nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
+nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1"
 
 # Build something content-addressed.
 outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link)
 
-nix path-info --json $outPathCA | jq -e '.[] | .ca | startswith("fixed:md5:")'
+nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")'
 
 # Content-addressed paths don't need signatures, so they verify
 # regardless of --sigs-needed.
-nix store verify $outPathCA
+nix store verify "$outPathCA"
-nix store verify $outPathCA --sigs-needed 1000
+nix store verify "$outPathCA" --sigs-needed 1000
 
 # Check that signing a content-addressed path doesn't overflow validSigs
-nix store sign --key-file $TEST_ROOT/sk1 $outPathCA
+nix store sign --key-file "$TEST_ROOT"/sk1 "$outPathCA"
-nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1
+nix store verify -r "$outPathCA" --sigs-needed 1000 --trusted-public-keys "$pk1"
 
 # Copy to a binary cache.
-nix copy --to file://$cacheDir $outPath2
+nix copy --to file://"$cacheDir" "$outPath2"
 
 # Verify that signatures got copied.
-info=$(nix path-info --store file://$cacheDir --json $outPath2)
+info=$(nix path-info --store file://"$cacheDir" --json "$outPath2")
-echo $info | jq -e '.[] | .ultimate == false'
+echo "$info" | jq -e '.[] | .ultimate == false'
-echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))'
+echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))'
-echo $info | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))'
+echo "$info" | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))'
 
 # Verify that adding a signature to a path in a binary cache works.
-nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2
+nix store sign --store file://"$cacheDir" --key-file "$TEST_ROOT"/sk2 "$outPath2"
-info=$(nix path-info --store file://$cacheDir --json $outPath2)
+info=$(nix path-info --store file://"$cacheDir" --json "$outPath2")
-echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))'
+echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))'
-echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))'
+echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))'
 
 # Copying to a diverted store should fail due to a lack of signatures by trusted keys.
-chmod -R u+w $TEST_ROOT/store0 || true
+chmod -R u+w "$TEST_ROOT"/store0 || true
-rm -rf $TEST_ROOT/store0
+rm -rf "$TEST_ROOT"/store0
 
 # Fails or very flaky only on GHA + macOS:
 #     expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key'
 # but this works:
-(! nix copy --to $TEST_ROOT/store0 $outPath)
+(! nix copy --to "$TEST_ROOT"/store0 "$outPath")
 
 # But succeed if we supply the public keys.
-nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1
+nix copy --to "$TEST_ROOT"/store0 "$outPath" --trusted-public-keys "$pk1"
 
-expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath
+expect 2 nix store verify --store "$TEST_ROOT"/store0 -r "$outPath"
 
-nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1
+nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --trusted-public-keys "$pk1"
-nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
+nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
 # It should also succeed if we disable signature checking.
-(! nix copy --to $TEST_ROOT/store0 $outPath2)
+(! nix copy --to "$TEST_ROOT"/store0 "$outPath2")
-nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2
+nix copy --to "$TEST_ROOT"/store0?require-sigs=false "$outPath2"
 
 # But signatures should still get copied.
-nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1
+nix store verify --store "$TEST_ROOT"/store0 -r "$outPath2" --trusted-public-keys "$pk1"
 
 # Content-addressed stuff can be copied without signatures.
-nix copy --to $TEST_ROOT/store0 $outPathCA
+nix copy --to "$TEST_ROOT"/store0 "$outPathCA"

tests/functional/simple.builder.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 echo "PATH=$PATH"
 
 # Verify that the PATH is empty.
@@ -5,7 +7,6 @@ if mkdir foo 2> /dev/null; then exit 1; fi
 
 # Set a PATH (!!! impure).
 export PATH=$goodPath
+mkdir "$out"
 
-mkdir $out
+echo "Hello World!" > "$out"/hello
-
-echo "Hello World!" > $out/hello

tests/functional/simple.sh

@@ -12,7 +12,7 @@ outPath=$(nix-store -rvv "$drvPath")
 
 echo "output path is $outPath"
 
-(! [ -w $outPath ])
+[[ ! -w $outPath ]]
 
 text=$(cat "$outPath/hello")
 if test "$text" != "Hello World!"; then exit 1; fi
@@ -21,16 +21,16 @@ TODO_NixOS
 
 # Directed delete: $outPath is not reachable from a root, so it should
 # be deleteable.
-nix-store --delete $outPath
+nix-store --delete "$outPath"
-(! [ -e $outPath/hello ])
+[[ ! -e $outPath/hello ]]
 
-outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)"
+outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode hash-check.nix)"
 if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then
     echo "hashDerivationModulo appears broken, got $outPath"
     exit 1
 fi
 
-outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode big-derivation-attr.nix)"
+outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode big-derivation-attr.nix)"
 if test "$outPath" != "/foo/xxiwa5zlaajv6xdjynf9yym9g319d6mn-big-derivation-attr.drv"; then
     echo "big-derivation-attr.nix hash appears broken, got $outPath. Memory corruption in large drv attr?"
     exit 1

tests/functional/ssh-relay.sh

@@ -2,10 +2,10 @@
 
 source common.sh
 
-echo foo > $TEST_ROOT/hello.sh
+echo foo > "$TEST_ROOT"/hello.sh
 
 ssh_localhost=ssh://localhost
-remote_store=?remote-store=$ssh_localhost
+remote_store="?remote-store=$ssh_localhost"
 
 store=$ssh_localhost
 
@@ -13,6 +13,6 @@ store+=$remote_store
 store+=$remote_store
 store+=$remote_store
 
-out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh)
+out=$(nix store add-path --store "$store" "$TEST_ROOT"/hello.sh)
 
-[ foo = $(< $out) ]
+[ foo = "$(< "$out")" ]

tests/functional/store-info.sh

@@ -8,12 +8,12 @@ STORE_INFO_JSON=$(nix store info --json)
 echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}"
 
 if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then
-    DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3)
+    DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3)
     echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION"
     [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]]
 fi
 
-expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \
+expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \
     fail "nix store info on a non-existent store should fail"
 
 TODO_NixOS

tests/functional/structured-attrs.sh

@@ -8,17 +8,19 @@ requireDaemonNewerThan "2.4pre20210712"
 
 clearStoreIfPossible
 
-rm -f $TEST_ROOT/result
+rm -f "$TEST_ROOT"/result
 
-nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
+nix-build structured-attrs.nix -A all -o "$TEST_ROOT"/result
 
-[[ $(cat $TEST_ROOT/result/foo) = bar ]]
+[[ $(cat "$TEST_ROOT"/result/foo) = bar ]]
-[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]]
+[[ $(cat "$TEST_ROOT"/result-dev/foo) = foo ]]
 
 export NIX_BUILD_SHELL=$SHELL
+# shellcheck disable=SC2016
 env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \
     --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
 
+# shellcheck disable=SC2016
 nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
 
 TODO_NixOS # following line fails.
@@ -26,6 +28,7 @@ TODO_NixOS # following line fails.
 # `nix develop` is a slightly special way of dealing with environment vars, it parses
 # these from a shell-file exported from a derivation. This is to test especially `outputs`
 # (which is an associative array in thsi case) being fine.
+# shellcheck disable=SC2016
 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"'
 
 nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE='

tests/functional/substitute-with-invalid-ca.sh

@@ -11,16 +11,16 @@ getRemoteNarInfo () {
     echo "$cacheDir/$(getHash "$1").narinfo"
 }
 
-cat <<EOF > $TEST_HOME/good.txt
+cat <<EOF > "$TEST_HOME"/good.txt
 I’m a good path
 EOF
 
-cat <<EOF > $TEST_HOME/bad.txt
+cat <<EOF > "$TEST_HOME"/bad.txt
 I’m a bad path
 EOF
 
-good=$(nix-store --add $TEST_HOME/good.txt)
+good=$(nix-store --add "$TEST_HOME"/good.txt)
-bad=$(nix-store --add $TEST_HOME/bad.txt)
+bad=$(nix-store --add "$TEST_HOME"/bad.txt)
 nix copy --to "$BINARY_CACHE" "$good"
 nix copy --to "$BINARY_CACHE" "$bad"
 nix-collect-garbage >/dev/null 2>&1

tests/functional/suggestions.sh

@@ -37,7 +37,7 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null)
 [[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \
     fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest"
 
-NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null)
+NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null)
 [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \
     fail "The evaluator should suggest the three closest possiblities"
 

tests/functional/tarball.sh

@@ -4,51 +4,51 @@ source common.sh
 
 clearStoreIfPossible
 
-rm -rf $TEST_HOME
+rm -rf "$TEST_HOME"
 
 tarroot=$TEST_ROOT/tarball
-rm -rf $tarroot
+rm -rf "$tarroot"
-mkdir -p $tarroot
+mkdir -p "$tarroot"
-cp dependencies.nix $tarroot/default.nix
+cp dependencies.nix "$tarroot/default.nix"
-cp config.nix dependencies.builder*.sh $tarroot/
+cp config.nix dependencies.builder*.sh "$tarroot/"
-touch -d '@1000000000' $tarroot $tarroot/*
+touch -d '@1000000000' "$tarroot" "$tarroot"/*
 
-hash=$(nix hash path $tarroot)
+hash=$(nix hash path "$tarroot")
 
 test_tarball() {
     local ext="$1"
     local compressor="$2"
 
     tarball=$TEST_ROOT/tarball.tar$ext
-    (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball
+    (cd "$TEST_ROOT" && GNUTAR_REPRODUCIBLE=1 tar --mtime="$tarroot"/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > "$tarball"
 
-    nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies
+    nix-env -f file://"$tarball" -qa --out-path | grepQuiet dependencies
 
-    nix-build -o $TEST_ROOT/result file://$tarball
+    nix-build -o "$TEST_ROOT"/result file://"$tarball"
 
-    nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
+    nix-build -o "$TEST_ROOT"/result '<foo>' -I foo=file://"$tarball"
 
-    nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
+    nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball file://$tarball)"
     # Do not re-fetch paths already present
-    nix-build  -o $TEST_ROOT/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })"
+    nix-build  -o "$TEST_ROOT"/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })"
 
-    nix-build  -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+    nix-build  -o "$TEST_ROOT"/result -E "import (fetchTree file://$tarball)"
-    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+    nix-build  -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
-    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+    nix-build  -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
     # Do not re-fetch paths already present
-    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })"
+    nix-build  -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })"
-    expectStderr 102 nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input'
+    expectStderr 102 nix-build  -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input'
 
     [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]]
 
     nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
     nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true'
 
-    nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar$ext
+    nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar"$ext"
-    nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file:///no-such-tarball$ext
+    nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file:///no-such-tarball"$ext"
-    (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file:///no-such-tarball$ext)
+    (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file:///no-such-tarball"$ext")
 
-    nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file:///no-such-tarball$ext -I fnord=.
+    nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file:///no-such-tarball"$ext" -I fnord=.
 
     # Ensure that the `name` attribute isn’t accepted as that would mess
     # with the content-addressing

tests/functional/test-infra.sh

@@ -14,7 +14,7 @@ expect 1 false
 expect 1 expect 0 false
 
 function ret() {
-  return $1
+  return "$1"
 }
 
 # `expect` can call functions, not just executables
@@ -48,6 +48,7 @@ expectStderr 1 noisyFalse | grepQuiet NAY
 
 # `set -o pipefile` is enabled
 
+# shellcheck disable=SC2317# shellcheck disable=SC2317
 pipefailure () {
     # shellcheck disable=SC2216
     true | false | true
@@ -55,6 +56,7 @@ pipefailure () {
 expect 1 pipefailure
 unset pipefailure
 
+# shellcheck disable=SC2317
 pipefailure () {
     # shellcheck disable=SC2216
     false | true | true
@@ -82,6 +84,7 @@ expect 1 useUnbound
 # ! alone unfortunately negates `set -e`, but it works in functions:
 # shellcheck disable=SC2251
 ! true
+# shellcheck disable=SC2317
 funBang () {
     ! true
 }

tests/functional/test-libstoreconsumer.sh

@@ -5,4 +5,4 @@ source common.sh
 drv="$(nix-instantiate simple.nix)"
 cat "$drv"
 out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")"
-cat "$out/hello" | grep -F "Hello World!"
+grep -F "Hello World!" < "$out/hello"

tests/functional/test-libstoreconsumer/meson.build

@@ -0,0 +1,14 @@
+libstoreconsumer_tester = executable(
+  'test-libstoreconsumer',
+  'main.cc',
+  cpp_args : [
+    # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+    # It would be nice for our headers to be idempotent instead.
+    '-include', 'config-util.hh',
+    '-include', 'config-store.hh',
+  ],
+  dependencies : [
+    dependency('nix-store'),
+  ],
+  build_by_default : false,
+)

tests/functional/timeout.sh

@@ -9,7 +9,7 @@ needLocalStore "see #4813"
 
 messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$?
 
-if [ $status -ne 101 ]; then
+if [ "$status" -ne 101 ]; then
     echo "error: 'nix-store' exited with '$status'; should have exited 101"
 
     # FIXME: https://github.com/NixOS/nix/issues/4813

tests/nixos/default.nix

@@ -4,20 +4,24 @@ let
 
   nixos-lib = import (nixpkgs + "/nixos/lib") { };
 
+  noTests = pkg: pkg.overrideAttrs (
+            finalAttrs: prevAttrs: {
+              doCheck = false;
+              doInstallCheck = false;
+            });
+
   # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
   runNixOSTestFor = system: test:
     (nixos-lib.runTest {
       imports = [
         test
-
-        # Add the quickBuild attribute to the check packages
-        ./quick-build.nix
       ];
 
       hostPkgs = nixpkgsFor.${system}.native;
       defaults = {
         nixpkgs.pkgs = nixpkgsFor.${system}.native;
         nix.checkAllErrors = false;
+        nix.package = noTests nixpkgsFor.${system}.native.nix;
       };
       _module.args.nixpkgs = nixpkgs;
       _module.args.system = system;
@@ -29,6 +33,9 @@ let
       forNix = nixVersion: runNixOSTestFor system {
         imports = [test];
         defaults.nixpkgs.overlays = [(curr: prev: {
+          # NOTE: noTests pkg might not have been built yet for some older versions of the package
+          #       and in versions before 2.25, the untested build wasn't shared with the tested build yet
+          #       Add noTests here when those versions become irrelevant.
           nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix;
         })];
       };

tests/nixos/quick-build.nix

@@ -1,47 +0,0 @@
-test@{ lib, extendModules, ... }:
-let
-  inherit (lib) mkOption types;
-in
-{
-  options = {
-    quickBuild = mkOption {
-      description = ''
-        Whether to perform a "quick" build of the Nix package to test.
-
-        When iterating on the functional tests, it's recommended to "set" this
-        to `true`, so that changes to the functional tests don't require any
-        recompilation of the package.
-        You can do so by building the `.quickBuild` attribute on the check package,
-        e.g:
-        ```console
-        nix build .#hydraJobs.functional_user.quickBuild
-        ```
-
-        We don't enable this by default to avoid the mostly unnecessary work of
-        performing an additional build of the package in cases where we build
-        the package normally anyway, such as in our pre-merge CI.
-      '';
-      type = types.bool;
-      default = false;
-    };
-  };
-
-  config = {
-    passthru.quickBuild = 
-      let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; };
-      in withQuickBuild.config.test;
-
-    defaults = { pkgs, ... }: {
-      config = lib.mkIf test.config.quickBuild {
-        nix.package = pkgs.nix_noTests;
-
-        system.forbiddenDependenciesRegexes = [
-          # This would indicate that the quickBuild feature is broken.
-          # It could happen if NixOS has a dependency on pkgs.nix instead of
-          # config.nix.package somewhere.
-          (builtins.unsafeDiscardStringContext pkgs.nix.outPath)
-        ];
-      };
-    };
-  };
-}

tests/unit/libexpr-support/local.mk

@@ -1,23 +0,0 @@
-libraries += libexpr-test-support
-
-libexpr-test-support_NAME = libnixexpr-test-support
-
-libexpr-test-support_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libexpr-test-support_INSTALL_DIR := $(checklibdir)
-else
-  libexpr-test-support_INSTALL_DIR :=
-endif
-
-libexpr-test-support_SOURCES := \
-    $(wildcard $(d)/tests/*.cc) \
-    $(wildcard $(d)/tests/value/*.cc)
-
-libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES)
-
-libexpr-test-support_LIBS = \
-    libstore-test-support libutil-test-support \
-    libexpr libstore libutil
-
-libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck

tests/unit/libexpr/local.mk

@@ -1,45 +0,0 @@
-check: libexpr-tests_RUN
-
-programs += libexpr-tests
-
-libexpr-tests_NAME := libnixexpr-tests
-
-libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml
-
-libexpr-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libexpr-tests_INSTALL_DIR := $(checkbindir)
-else
-  libexpr-tests_INSTALL_DIR :=
-endif
-
-libexpr-tests_SOURCES := \
-    $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc) \
-    $(wildcard $(d)/flake/*.cc)
-
-libexpr-tests_EXTRA_INCLUDES = \
-    -I tests/unit/libexpr-support \
-    -I tests/unit/libstore-support \
-    -I tests/unit/libutil-support \
-    $(INCLUDE_libexpr) \
-    $(INCLUDE_libexprc) \
-    $(INCLUDE_libfetchers) \
-    $(INCLUDE_libstore) \
-    $(INCLUDE_libstorec) \
-    $(INCLUDE_libutil) \
-    $(INCLUDE_libutilc)
-
-libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES)
-
-libexpr-tests_LIBS = \
-    libexpr-test-support libstore-test-support libutil-test-support \
-    libexpr libexprc libfetchers libstore libstorec libutil libutilc
-
-libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock
-
-ifdef HOST_WINDOWS
-  # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space
-  libexpr-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024)))
-endif

tests/unit/libfetchers/local.mk

@@ -1,37 +0,0 @@
-check: libfetchers-tests_RUN
-
-programs += libfetchers-tests
-
-libfetchers-tests_NAME = libnixfetchers-tests
-
-libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml
-
-libfetchers-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libfetchers-tests_INSTALL_DIR := $(checkbindir)
-else
-  libfetchers-tests_INSTALL_DIR :=
-endif
-
-libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libfetchers-tests_EXTRA_INCLUDES = \
-    -I tests/unit/libstore-support \
-    -I tests/unit/libutil-support \
-    $(INCLUDE_libfetchers) \
-    $(INCLUDE_libstore) \
-    $(INCLUDE_libutil)
-
-libfetchers-tests_CXXFLAGS += $(libfetchers-tests_EXTRA_INCLUDES)
-
-libfetchers-tests_LIBS = \
-    libstore-test-support libutil-test-support \
-    libfetchers libstore libutil
-
-libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS)
-
-ifdef HOST_WINDOWS
-  # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space
-  libfetchers-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024)))
-endif

tests/unit/libflake/local.mk

@@ -1,43 +0,0 @@
-check: libflake-tests_RUN
-
-programs += libflake-tests
-
-libflake-tests_NAME := libnixflake-tests
-
-libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml
-
-libflake-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libflake-tests_INSTALL_DIR := $(checkbindir)
-else
-  libflake-tests_INSTALL_DIR :=
-endif
-
-libflake-tests_SOURCES := \
-    $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc) \
-    $(wildcard $(d)/flake/*.cc)
-
-libflake-tests_EXTRA_INCLUDES = \
-    -I tests/unit/libflake-support \
-    -I tests/unit/libstore-support \
-    -I tests/unit/libutil-support \
-    $(INCLUDE_libflake) \
-    $(INCLUDE_libexpr) \
-    $(INCLUDE_libfetchers) \
-    $(INCLUDE_libstore) \
-    $(INCLUDE_libutil) \
-
-libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES)
-
-libflake-tests_LIBS = \
-    libexpr-test-support libstore-test-support libutil-test-support \
-    libflake libexpr libfetchers libstore libutil
-
-libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock
-
-ifdef HOST_WINDOWS
-  # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space
-  libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024)))
-endif

tests/unit/libstore-support/local.mk

@@ -1,21 +0,0 @@
-libraries += libstore-test-support
-
-libstore-test-support_NAME = libnixstore-test-support
-
-libstore-test-support_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-test-support_INSTALL_DIR := $(checklibdir)
-else
-  libstore-test-support_INSTALL_DIR :=
-endif
-
-libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
-
-libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES)
-
-libstore-test-support_LIBS = \
-    libutil-test-support \
-    libstore libutil
-
-libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck

tests/unit/libstore/local.mk

@@ -1,38 +0,0 @@
-check: libstore-tests_RUN
-
-programs += libstore-tests
-
-libstore-tests_NAME = libnixstore-tests
-
-libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml
-
-libstore-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-tests_INSTALL_DIR := $(checkbindir)
-else
-  libstore-tests_INSTALL_DIR :=
-endif
-
-libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libstore-tests_EXTRA_INCLUDES = \
-    -I tests/unit/libstore-support \
-    -I tests/unit/libutil-support \
-    $(INCLUDE_libstore) \
-    $(INCLUDE_libstorec) \
-    $(INCLUDE_libutil) \
-    $(INCLUDE_libutilc)
-
-libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES)
-
-libstore-tests_LIBS = \
-    libstore-test-support libutil-test-support \
-    libstore libstorec libutil libutilc
-
-libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
-
-ifdef HOST_WINDOWS
-  # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space
-  libstore-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024)))
-endif

tests/unit/libutil-support/local.mk

@@ -1,19 +0,0 @@
-libraries += libutil-test-support
-
-libutil-test-support_NAME = libnixutil-test-support
-
-libutil-test-support_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libutil-test-support_INSTALL_DIR := $(checklibdir)
-else
-  libutil-test-support_INSTALL_DIR :=
-endif
-
-libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
-
-libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
-
-libutil-test-support_LIBS = libutil
-
-libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck

tests/unit/libutil/local.mk

@@ -1,37 +0,0 @@
-check: libutil-tests_RUN
-
-programs += libutil-tests
-
-libutil-tests_NAME = libnixutil-tests
-
-libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml
-
-libutil-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libutil-tests_INSTALL_DIR := $(checkbindir)
-else
-  libutil-tests_INSTALL_DIR :=
-endif
-
-libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libutil-tests_EXTRA_INCLUDES = \
-    -I tests/unit/libutil-support \
-    $(INCLUDE_libutil) \
-    $(INCLUDE_libutilc)
-
-libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
-
-libutil-tests_LIBS = libutil-test-support libutil libutilc
-
-libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
-
-ifdef HOST_WINDOWS
-  # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space
-  libutil-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024)))
-endif
-
-check: $(d)/data/git/check-data.sh.test
-
-$(eval $(call run-test,$(d)/data/git/check-data.sh))