Merge branch 'master' into nixenvjsondrvpath

.editorconfig

@@ -17,7 +17,7 @@ indent_style = space
 indent_size = 2
 
 # Match c++/shell/perl, set indent to spaces with width of four
-[*.{hpp,cc,hh,sh,pl}]
+[*.{hpp,cc,hh,sh,pl,xs}]
 indent_style = space
 indent_size = 4
 

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v1.4.0
+        uses: zeebe-io/backport-action@v2.1.1
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

Makefile.config.in

@@ -28,6 +28,8 @@ SODIUM_LIBS = @SODIUM_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 bash = @bash@
 bindir = @bindir@
+checkbindir = @checkbindir@
+checklibdir = @checklibdir@
 datadir = @datadir@
 datarootdir = @datarootdir@
 doc_generate = @doc_generate@
@@ -48,4 +50,5 @@ sysconfdir = @sysconfdir@
 system = @system@
 ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_TESTS = @ENABLE_TESTS@
+INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@
 internal_api_docs = @internal_api_docs@

configure.ac

@@ -68,6 +68,9 @@ case "$host_os" in
 esac
 
 
+ENSURE_NO_GCC_BUG_80431
+
+
 # Check for pubsetbuf.
 AC_MSG_CHECKING([for pubsetbuf])
 AC_LANG_PUSH(C++)
@@ -164,6 +167,18 @@ AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
   ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
 AC_SUBST(ENABLE_TESTS)
 
+AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]),
+  INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no)
+AC_SUBST(INSTALL_UNIT_TESTS)
+
+AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]),
+  checkbindir=$withval, checkbindir=$libexecdir/nix)
+AC_SUBST(checkbindir)
+
+AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]),
+  checklibdir=$withval, checklibdir=$libdir)
+AC_SUBST(checklibdir)
+
 # Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
 AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
   internal_api_docs=$enableval, internal_api_docs=no)

doc/manual/local.mk

@@ -103,7 +103,7 @@ $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix

doc/manual/src/SUMMARY.md.in

@@ -18,6 +18,8 @@
   - [Uninstalling Nix](installation/uninstall.md)
 - [Nix Store](store/index.md)
   - [File System Object](store/file-system-object.md)
+  - [Store Object](store/store-object.md)
+  - [Store Path](store/store-path.md)
 - [Nix Language](language/index.md)
   - [Data Types](language/values.md)
   - [Language Constructs](language/constructs.md)
@@ -113,6 +115,7 @@
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
   - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)

doc/manual/src/architecture/architecture.md

@@ -63,7 +63,7 @@ The command line interface and Nix expressions are what users deal with most.
 > The Nix language itself does not have a notion of *packages* or *configurations*.
 > As far as we are concerned here, the inputs and results of a build plan are just data.
 
-Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them.
+Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them.
 It can also execute build plans to produce new data, which are made available to the operating system as files.
 
 A build plan itself is a series of *build tasks*, together with their build inputs.

doc/manual/src/contributing/documentation.md

@@ -162,6 +162,24 @@ Please observe these guidelines to ease reviews:
   > This is a note.
   ```
 
+  Highlight examples as such:
+
+  ````
+  > **Example**
+  >
+  > ```console
+  > $ nix --version
+  > ```
+  ````
+
+  Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation:
+
+  ````
+  > **Syntax**
+  >
+  > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}`
+  ````
+
 ### The `@docroot@` variable
 
 `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.

doc/manual/src/contributing/hacking.md

@@ -210,7 +210,7 @@ See [supported compilation environments](#compilation-environments) and instruct
 To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running:
 
 ```console
-make clean && bear -- make -j$NIX_BUILD_CORES install
+make clean && bear -- make -j$NIX_BUILD_CORES default check install
 ```
 
 Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration).

doc/manual/src/contributing/testing.md

@@ -133,17 +133,17 @@ ran test tests/functional/${testName}.sh... [PASS]
 or without `make`:
 
 ```shell-session
-$ ./mk/run-test.sh tests/functional/${testName}.sh
+$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh
 ran test tests/functional/${testName}.sh... [PASS]
 ```
 
 To see the complete output, one can also run:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/functional/${testName}.sh
+$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh
-+ foo
++(${testName}.sh:1) foo
 output from foo
-+ bar
++(${testName}.sh:2) bar
 output from bar
 ...
 ```
@@ -175,7 +175,7 @@ edit it like so:
 Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/functional/${testName}.sh
+$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh
 ...
 + gdb blash blub
 GNU gdb (GDB) 12.1

doc/manual/src/glossary.md

@@ -59,7 +59,7 @@
 - [store]{#gloss-store}
 
   A collection of store objects, with operations to manipulate that collection.
-  See [Nix Store] for details.
+  See [Nix store](./store/index.md) for details.
 
   There are many types of stores.
   See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list.
@@ -86,10 +86,13 @@
 
 - [store path]{#gloss-store-path}
 
-  The location of a [store object] in the file system, i.e., an
+  The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory.
-  immediate child of the Nix store directory.
 
-  Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
+  > **Example**
+  >
+  > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
+
+  See [Store Path](@docroot@/store/store-path.md) for details.
 
   [store path]: #gloss-store-path
 

doc/manual/src/language/constructs.md

@@ -132,6 +132,32 @@ a = src-set.a; b = src-set.b; c = src-set.c;
 when used while defining local variables in a let-expression or while
 defining a set.
 
+In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example
+
+
+```nix
+let
+  x = { a = 1; b = 2; };
+  inherit (builtins) attrNames;
+in
+{
+  names = attrNames x;
+}
+```
+
+is equivalent to
+
+```nix
+let
+  x = { a = 1; b = 2; };
+in
+{
+  names = builtins.attrNames x;
+}
+```
+
+both evaluate to `{ names = [ "a" "b" ]; }`.
+
 ## Functions
 
 Functions have the following form:
@@ -146,65 +172,65 @@ three kinds of patterns:
 
   - If a pattern is a single identifier, then the function matches any
     argument. Example:
-    
+
     ```nix
     let negate = x: !x;
         concat = x: y: x + y;
     in if negate true then concat "foo" "bar" else ""
     ```
-    
+
     Note that `concat` is a function that takes one argument and returns
     a function that takes another argument. This allows partial
     parameterisation (i.e., only filling some of the arguments of a
     function); e.g.,
-    
+
     ```nix
     map (concat "foo") [ "bar" "bla" "abc" ]
     ```
-    
+
     evaluates to `[ "foobar" "foobla" "fooabc" ]`.
 
   - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a
     set containing the listed attributes, and binds the values of those
     attributes to variables in the function body. For example, the
     function
-    
+
     ```nix
     { x, y, z }: z + y + x
     ```
-    
+
     can only be called with a set containing exactly the attributes `x`,
     `y` and `z`. No other attributes are allowed. If you want to allow
     additional arguments, you can use an ellipsis (`...`):
-    
+
     ```nix
     { x, y, z, ... }: z + y + x
     ```
-    
+
     This works on any set that contains at least the three named
     attributes.
-    
+
     It is possible to provide *default values* for attributes, in
     which case they are allowed to be missing. A default value is
     specified by writing `name ?  e`, where *e* is an arbitrary
     expression. For example,
-    
+
     ```nix
     { x, y ? "foo", z ? "bar" }: z + y + x
     ```
-    
+
     specifies a function that only requires an attribute named `x`, but
     optionally accepts `y` and `z`.
 
   - An `@`-pattern provides a means of referring to the whole value
     being matched:
-    
+
     ```nix
     args@{ x, y, z, ... }: z + y + x + args.a
     ```
-    
+
     but can also be written as:
-    
+
     ```nix
     { x, y, z, ... } @ args: z + y + x + args.a
     ```

doc/manual/src/language/operators.md

@@ -25,7 +25,7 @@
 | Inequality                             | *expr* `!=` *expr*                         | none          | 11         |
 | Logical conjunction (`AND`)            | *bool* `&&` *bool*                         | left          | 12         |
 | Logical disjunction (`OR`)             | *bool* <code>\|\|</code> *bool*            | left          | 13         |
-| [Logical implication]                  | *bool* `->` *bool*                         | none          | 14         |
+| [Logical implication]                  | *bool* `->` *bool*                         | right         | 14         |
 
 [string]: ./values.md#type-string
 [path]: ./values.md#type-path

doc/manual/src/release-notes/rl-2.19.md

@@ -0,0 +1,77 @@
+# Release 2.19 (2023-11-17)
+
+- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter)
+  by appending the contents of any `#! nix` lines and the script's location into a single call.
+
+- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters.
+
+- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`).
+
+- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly.
+
+- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.<SYSTEM>` or `legacyPackages.<SYSTEM>`.
+
+- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts.
+
+- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
+
+- `nix-shell` shebang lines now support single-quoted arguments.
+
+- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
+  As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes.
+
+- The interface for creating and updating lock files has been overhauled:
+
+  - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
+    It will *never* update existing inputs.
+
+  - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
+    - Passing no arguments will update all inputs of the current flake, just like it already did.
+    - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
+    - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
+
+  - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
+    They are superceded by `nix flake update`.
+
+- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md)
+  (experimental) now returns a JSON map rather than JSON list.
+  The `path` field of each object has instead become the key in the outer map, since it is unique.
+  The `valid` field also goes away because we just use `null` instead.
+
+  - Old way:
+
+    ```json5
+    [
+      {
+        "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15",
+        "valid": true,
+        // ...
+      },
+      {
+        "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path",
+        "valid": false
+      }
+    ]
+    ```
+
+  - New way
+
+    ```json5
+    {
+      "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": {
+        // ...
+      },
+      "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null,
+    }
+    ```
+
+  This makes it match `nix derivation show`, which also maps store paths to information.
+
+- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish)
+  [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile.
+  This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md)
+  (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
+
+- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated.

doc/manual/src/release-notes/rl-next.md

@@ -1,19 +1,3 @@
 # Release X.Y (202?-??-??)
 
-- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters.
+- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`).
-
-- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly.
-
-- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.<SYSTEM>` or `legacyPackages.<SYSTEM>`.
-
-- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts.
-
-- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
-
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-- `nix-shell` shebang lines now support single-quoted arguments.
-
-- `builtins.fetchTree` is now marked as stable.

doc/manual/src/store/index.md

@@ -1,4 +1,5 @@
 # Nix Store
 
-The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them.
+The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data.
-There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches.
+
+There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches.

doc/manual/src/store/store-object.md

@@ -0,0 +1,10 @@
+## Store Object
+
+A Nix store is a collection of *store objects* with *references* between them.
+A store object consists of
+
+  - A [file system object](./file-system-object.md) as data
+  - A set of [store paths](./store-path.md) as references to other store objects
+
+Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object):
+Once created, they do not change until they are deleted.

doc/manual/src/store/store-path.md

@@ -0,0 +1,69 @@
+# Store Path
+
+Nix implements references to [store objects](./index.md#store-object) as *store paths*.
+
+Think of a store path as an [opaque], [unique identifier]:
+The only way to obtain store path is by adding or building store objects.
+A store path will always reference exactly one store object.
+
+[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type
+[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier
+
+Store paths are pairs of
+
+- A 20-byte digest for identification
+- A symbolic name for people to read
+
+> **Example**
+>
+> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z`
+> - Name:   `firefox-33.1`
+
+To make store objects accessible to operating system processes, stores have to expose store objects through the file system.
+
+A store path is rendered to a file system path as the concatenation of
+
+- [Store directory](#store-directory) (typically `/nix/store`)
+- Path separator (`/`)
+- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
+- Hyphen (`-`)
+- Name
+
+> **Example**
+>
+> ```
+>   /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1
+>   |--------| |------------------------------| |----------|
+> store directory            digest                 name
+> ```
+
+## Store Directory
+
+Every [Nix store](./index.md) has a store directory.
+
+Not every store can be accessed through the file system.
+But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path).
+
+[file system objects]: ./file-system-object.md
+
+This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
+
+> **Note**
+>
+> The store directory defaults to `/nix/store`, but is in principle arbitrary.
+
+It is important which store a given store object belongs to:
+Files in the store object can contain store paths, and processes may read these paths.
+Nix can only guarantee referential integrity if store paths do not cross store boundaries.
+
+Therefore one can only copy store objects to a different store if
+
+- The source and target stores' directories match
+
+  or
+
+- The store object in question has no references, that is, contains no store paths
+
+One cannot copy a store object to a store with a different store directory.
+Instead, it has to be rebuilt, together with all its dependencies.
+It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums.

flake.lock

@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1695283060,
+        "lastModified": 1698876495,
-        "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=",
+        "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4",
+        "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "release-23.05",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,7 +1,9 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  # FIXME go back to nixos-23.05-small once
+  # https://github.com/NixOS/nixpkgs/pull/264875 is included.
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
@@ -162,6 +164,10 @@
 
         testConfigureFlags = [
           "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
+        ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+          "--enable-install-unit-tests"
+          "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
+          "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
         ];
 
         internalApiDocsConfigureFlags = [
@@ -183,6 +189,7 @@
             buildPackages.git
             buildPackages.mercurial # FIXME: remove? only needed for tests
             buildPackages.jq # Also for custom mdBook preprocessor.
+            buildPackages.openssh # only needed for tests (ssh-keygen)
           ]
           ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
 
@@ -401,7 +408,8 @@
             src = nixSrc;
             VERSION_SUFFIX = versionSuffix;
 
-            outputs = [ "out" "dev" "doc" ];
+            outputs = [ "out" "dev" "doc" ]
+              ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check";
 
             nativeBuildInputs = nativeBuildDeps;
             buildInputs = buildDeps
@@ -707,7 +715,8 @@
           stdenv.mkDerivation {
             name = "nix";
 
-            outputs = [ "out" "dev" "doc" ];
+            outputs = [ "out" "dev" "doc" ]
+              ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check";
 
             nativeBuildInputs = nativeBuildDeps
               ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear

m4/gcc_bug_80431.m4

@@ -0,0 +1,64 @@
+# Ensure that this bug is not present in the C++ toolchain we are using.
+#
+# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431
+#
+# The test program is from that issue, with only a slight modification
+# to set an exit status instead of printing strings.
+AC_DEFUN([ENSURE_NO_GCC_BUG_80431],
+[
+  AC_MSG_CHECKING([that GCC bug 80431 is fixed])
+  AC_LANG_PUSH(C++)
+  AC_RUN_IFELSE(
+    [AC_LANG_PROGRAM(
+      [[
+        #include <cstdio>
+
+        static bool a = true;
+        static bool b = true;
+
+        struct Options { };
+
+        struct Option
+        {
+            Option(Options * options)
+            {
+                a = false;
+            }
+
+            ~Option()
+            {
+                b = false;
+            }
+        };
+
+        struct MyOptions : Options { };
+
+        struct MyOptions2 : virtual MyOptions
+        {
+            Option foo{this};
+        };
+      ]],
+      [[
+        {
+            MyOptions2 opts;
+        }
+        return (a << 1) | b;
+      ]])],
+    [status_80431=0],
+    [status_80431=$?],
+    [
+      # Assume we're bug-free when cross-compiling
+    ])
+  AC_LANG_POP(C++)
+  AS_CASE([$status_80431],
+    [0],[
+      AC_MSG_RESULT(yes)
+    ],
+    [2],[
+      AC_MSG_RESULT(no)
+      AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug)
+    ],
+    [
+      AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring)
+    ])
+])

maintainers/README.md

@@ -2,7 +2,7 @@
 
 ## Motivation
 
-The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape.
+The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape.
 
 We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs.
 

mk/common-test.sh

@@ -1,15 +1,27 @@
-test_dir=tests/functional
+# Remove overall test dir (at most one of the two should match) and
+# remove file extension.
+test_name=$(echo -n "$test" | sed \
+    -e "s|^unit-test-data/||" \
+    -e "s|^tests/functional/||" \
+    -e "s|\.sh$||" \
+    )
 
-test=$(echo -n "$test" | sed -e "s|^$test_dir/||")
+TESTS_ENVIRONMENT=(
-
+    "TEST_NAME=$test_name"
-TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
+    'NIX_REMOTE='
+    'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) '
+)
 
 : ${BASH:=/usr/bin/env bash}
 
+run () {
+   cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1)
+}
+
 init_test () {
-   cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
+   run "$init" 2>/dev/null > /dev/null
 }
 
 run_test_proper () {
-   cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
+   run "$test"
 }

mk/debug-test.sh

@@ -3,9 +3,12 @@
 set -eu -o pipefail
 
 test=$1
+init=${2-}
 
 dir="$(dirname "${BASH_SOURCE[0]}")"
 source "$dir/common-test.sh"
 
-(init_test)
+if [ -n "$init" ]; then
+    (init_test)
+fi
 run_test_proper

mk/lib.mk

@@ -122,14 +122,15 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
 $(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
 $(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
 $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
+install_test_init=tests/functional/init.sh
 $(foreach test, $(install-tests), \
-  $(eval $(call run-install-test,$(test))) \
+  $(eval $(call run-test,$(test),$(install_test_init))) \
   $(eval installcheck: $(test).test))
 $(foreach test-group, $(install-tests-groups), \
-  $(eval $(call run-install-test-group,$(test-group))) \
+  $(eval $(call run-test-group,$(test-group),$(install_test_init))) \
   $(eval installcheck: $(test-group).test-group) \
   $(foreach test, $($(test-group)-tests), \
-    $(eval $(call run-install-test,$(test))) \
+    $(eval $(call run-test,$(test),$(install_test_init))) \
     $(eval $(test-group).test-group: $(test).test)))
 
 $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))

mk/run-test.sh

@@ -8,6 +8,7 @@ yellow=""
 normal=""
 
 test=$1
+init=${2-}
 
 dir="$(dirname "${BASH_SOURCE[0]}")"
 source "$dir/common-test.sh"
@@ -21,7 +22,9 @@ if [ -t 1 ]; then
 fi
 
 run_test () {
-    (init_test 2>/dev/null > /dev/null)
+    if [ -n "$init" ]; then
+        (init_test 2>/dev/null > /dev/null)
+    fi
     log="$(run_test_proper 2>&1)" && status=0 || status=$?
 }
 

mk/tests.mk

@@ -2,19 +2,22 @@
 
 test-deps =
 
-define run-install-test
+define run-bash
 
-  .PHONY: $1.test
+  .PHONY: $1
-  $1.test: $1 $(test-deps)
+  $1: $2
-	@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
+	@env BASH=$(bash) $(bash) $3 < /dev/null
-
-  .PHONY: $1.test-debug
-  $1.test-debug: $1 $(test-deps)
-	@env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null
 
 endef
 
-define run-install-test-group
+define run-test
+
+  $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2))
+  $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2))
+
+endef
+
+define run-test-group
 
   .PHONY: $1.test-group
 

perl/lib/Nix/Store.xs

@@ -11,7 +11,6 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "store-api.hh"
-#include "util.hh"
 #include "crypto.hh"
 
 #include <sodium.h>

scripts/nix-profile-daemon.fish.in

@@ -19,6 +19,14 @@ set __ETC_PROFILE_NIX_SOURCED 1
 
 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
+# Populate bash completions, .desktop files, etc
+if test -z "$XDG_DATA_DIRS"
+    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+    set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share"
+else
+    set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share"
+end
+
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if test -n "$NIX_SSH_CERT_FILE"
   : # Allow users to override the NIX_SSL_CERT_FILE

scripts/nix-profile-daemon.sh.in

@@ -30,6 +30,14 @@ fi
 
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
+# Populate bash completions, .desktop files, etc
+if [ -z "$XDG_DATA_DIRS" ]; then
+    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+    export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+else
+    export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+fi
+
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE

scripts/nix-profile.fish.in

@@ -20,6 +20,14 @@ if test -n "$HOME" && test -n "$USER"
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
+    # Populate bash completions, .desktop files, etc
+    if test -z "$XDG_DATA_DIRS"
+        # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+        set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    else
+        set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    end
+
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if test -n "$NIX_SSH_CERT_FILE"
         : # Allow users to override the NIX_SSL_CERT_FILE

scripts/nix-profile.sh.in

@@ -32,6 +32,14 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
+    # Populate bash completions, .desktop files, etc
+    if [ -z "$XDG_DATA_DIRS" ]; then
+        # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
+        export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    else
+        export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
+    fi
+
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
         export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

src/libcmd/built-path.hh

@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include "derived-path.hh"
 #include "realisation.hh"
 

src/libcmd/command.cc

@@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.push_back(BuiltPath::Opaque{p});
+            paths.emplace_back(BuiltPath::Opaque{p});
     } else {
         paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
@@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
             }
             store->computeFSClosure(pathsRoots, pathsClosure);
             for (auto & path : pathsClosure)
-                paths.push_back(BuiltPath::Opaque{path});
+                paths.emplace_back(BuiltPath::Opaque{path});
         }
     }
 

src/libcmd/command.hh

@@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args {
     void setEnviron();
 };
 
+void completeFlakeInputPath(
+    AddCompletions & completions,
+    ref<EvalState> evalState,
+    const std::vector<FlakeRef> & flakeRefs,
+    std::string_view prefix);
+
 void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
 
 void completeFlakeRefWithFragment(

src/libcmd/common-eval-args.cc

@@ -2,7 +2,6 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
 #include "filetransfer.hh"
-#include "util.hh"
 #include "eval.hh"
 #include "fetchers.hh"
 #include "registry.hh"
@@ -165,7 +164,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
     return res.finish();
 }
 
-SourcePath lookupFileArg(EvalState & state, std::string_view s)
+SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir)
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto storePath = fetchers::downloadTarball(
@@ -186,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s)
     }
 
     else
-        return state.rootPath(CanonPath::fromCwd(s));
+        return state.rootPath(CanonPath(s, baseDir));
 }
 
 }

src/libcmd/common-eval-args.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "args.hh"
+#include "canon-path.hh"
 #include "common-args.hh"
 #include "search-path.hh"
 
@@ -28,6 +29,6 @@ private:
     std::map<std::string, std::string> autoArgs;
 };
 
-SourcePath lookupFileArg(EvalState & state, std::string_view s);
+SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd());
 
 }

src/libcmd/editor-for.cc

@@ -1,5 +1,5 @@
-#include "util.hh"
 #include "editor-for.hh"
+#include "environment-variables.hh"
 
 namespace nix {
 

src/libcmd/installable-attr-path.hh

@@ -4,7 +4,6 @@
 #include "globals.hh"
 #include "installable-value.hh"
 #include "outputs-spec.hh"
-#include "util.hh"
 #include "command.hh"
 #include "attr-path.hh"
 #include "common-eval-args.hh"

src/libcmd/installables.cc

@@ -4,6 +4,7 @@
 #include "installable-attr-path.hh"
 #include "installable-flake.hh"
 #include "outputs-spec.hh"
+#include "users.hh"
 #include "util.hh"
 #include "command.hh"
 #include "attr-path.hh"
@@ -28,7 +29,7 @@
 
 namespace nix {
 
-static void completeFlakeInputPath(
+void completeFlakeInputPath(
     AddCompletions & completions,
     ref<EvalState> evalState,
     const std::vector<FlakeRef> & flakeRefs,
@@ -46,13 +47,6 @@ MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
 
-    addFlag({
-        .longName = "recreate-lock-file",
-        .description = "Recreate the flake's lock file from scratch.",
-        .category = category,
-        .handler = {&lockFlags.recreateLockFile, true}
-    });
-
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
@@ -85,19 +79,6 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {&lockFlags.commitLockFile, true}
     });
 
-    addFlag({
-        .longName = "update-input",
-        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
-        .category = category,
-        .labels = {"input-path"},
-        .handler = {[&](std::string s) {
-            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
-        }},
-        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
-        }}
-    });
-
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
@@ -107,7 +88,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath("."), true));
+                parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
@@ -149,7 +130,7 @@ MixFlakeOptions::MixFlakeOptions()
             auto evalState = getEvalState();
             auto flake = flake::lockFlake(
                 *evalState,
-                parseFlakeRef(flakeRef, absPath(".")),
+                parseFlakeRef(flakeRef, absPath(getCommandBaseDir())),
                 { .writeLockFile = false });
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
@@ -313,6 +294,8 @@ void completeFlakeRefWithFragment(
                 prefixRoot = ".";
             }
             auto flakeRefS = std::string(prefix.substr(0, hash));
+
+            // TODO: ideally this would use the command base directory instead of assuming ".".
             auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
@@ -461,10 +444,12 @@ Installables SourceExprCommand::parseInstallables(
             auto e = state->parseStdin();
             state->eval(e, *vFile);
         }
-        else if (file)
+        else if (file) {
-            state->evalFile(lookupFileArg(*state, *file), *vFile);
+            state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile);
+        }
         else {
-            auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
+            CanonPath dir(CanonPath::fromCwd(getCommandBaseDir()));
+            auto e = state->parseExprFromString(*expr, state->rootPath(dir));
             state->eval(e, *vFile);
         }
 
@@ -499,7 +484,7 @@ Installables SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath("."));
+                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir()));
                 result.push_back(make_ref<InstallableFlake>(
                         this,
                         getEvalState(),
@@ -683,7 +668,7 @@ BuiltPaths Installable::toBuiltPaths(
 
         BuiltPaths res;
         for (auto & drvPath : Installable::toDerivations(store, installables, true))
-            res.push_back(BuiltPath::Opaque{drvPath});
+            res.emplace_back(BuiltPath::Opaque{drvPath});
         return res;
     }
 }
@@ -773,7 +758,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     for (auto i : rawInstallables)
         res.push_back(parseFlakeRefWithFragment(
             expandTilde(i),
-            absPath(".")).first);
+            absPath(getCommandBaseDir())).first);
     return res;
 }
 
@@ -795,7 +780,7 @@ std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
     return {
         parseFlakeRefWithFragment(
             expandTilde(_installable),
-            absPath(".")).first
+            absPath(getCommandBaseDir())).first
     };
 }
 

src/libcmd/installables.hh

@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "util.hh"
 #include "path.hh"
 #include "outputs-spec.hh"
 #include "derived-path.hh"

src/libcmd/markdown.cc

@@ -1,6 +1,7 @@
 #include "markdown.hh"
 #include "util.hh"
 #include "finally.hh"
+#include "terminal.hh"
 
 #include <sys/queue.h>
 #include <lowdown.h>

src/libcmd/repl.cc

@@ -22,6 +22,7 @@ extern "C" {
 #include "repl.hh"
 
 #include "ansicolor.hh"
+#include "signals.hh"
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-cache.hh"
@@ -36,6 +37,8 @@ extern "C" {
 #include "globals.hh"
 #include "flake/flake.hh"
 #include "flake/lockfile.hh"
+#include "users.hh"
+#include "terminal.hh"
 #include "editor-for.hh"
 #include "finally.hh"
 #include "markdown.hh"

src/libexpr/attr-path.cc

@@ -1,6 +1,5 @@
 #include "attr-path.hh"
 #include "eval-inline.hh"
-#include "util.hh"
 
 
 namespace nix {

src/libexpr/eval-cache.cc

@@ -1,3 +1,4 @@
+#include "users.hh"
 #include "eval-cache.hh"
 #include "sqlite.hh"
 #include "eval.hh"

src/libexpr/eval-settings.cc

@@ -1,3 +1,4 @@
+#include "users.hh"
 #include "globals.hh"
 #include "profiles.hh"
 #include "eval.hh"

src/libexpr/eval-settings.hh

@@ -1,4 +1,6 @@
 #pragma once
+///@file
+
 #include "config.hh"
 
 namespace nix {

src/libexpr/eval.cc

@@ -1,6 +1,7 @@
 #include "eval.hh"
 #include "eval-settings.hh"
 #include "hash.hh"
+#include "primops.hh"
 #include "types.hh"
 #include "util.hh"
 #include "store-api.hh"
@@ -14,6 +15,7 @@
 #include "print.hh"
 #include "fs-input-accessor.hh"
 #include "memory-input-accessor.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -29,6 +31,7 @@
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
+#include <boost/container/small_vector.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -721,6 +724,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
 }
 
 
+void PrimOp::check()
+{
+    if (arity > maxPrimOpArity) {
+        throw Error("primop arity must not exceed %1%", maxPrimOpArity);
+    }
+}
+
+
+void Value::mkPrimOp(PrimOp * p)
+{
+    p->check();
+    clearValue();
+    internalType = tPrimOp;
+    primOp = p;
+}
+
+
 Value * EvalState::addPrimOp(PrimOp && primOp)
 {
     /* Hack to make constants lazy: turn them into a application of
@@ -1690,7 +1710,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                Value * vArgs[arity];
+                Value * vArgs[maxPrimOpArity];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
@@ -1747,11 +1767,17 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     Value vFun;
     fun->eval(state, env, vFun);
 
-    Value * vArgs[args.size()];
+    // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5}
+    // 2: over 4000
+    // 3: about 300
+    // 4: about 60
+    // 5: under 10
+    // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
+    boost::container::small_vector<Value *, 4> vArgs(args.size());
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs, v, pos);
+    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
 }
 
 
@@ -1990,8 +2016,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    Value values[es->size()];
+    boost::container::small_vector<Value, conservativeStackReservation> values(es->size());
-    Value * vTmpP = values;
+    Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
         Value & vTmp = *vTmpP++;

src/libexpr/eval.hh

@@ -18,6 +18,12 @@
 
 namespace nix {
 
+/**
+ * We put a limit on primop arity because it lets us use a fixed size array on
+ * the stack. 8 is already an impractical number of arguments. Use an attrset
+ * argument for such overly complicated functions.
+ */
+constexpr size_t maxPrimOpArity = 8;
 
 class Store;
 class EvalState;
@@ -71,6 +77,12 @@ struct PrimOp
      * Optional experimental for this to be gated on.
      */
     std::optional<ExperimentalFeature> experimentalFeature;
+
+    /**
+     * Validity check to be performed by functions that introduce primops,
+     * such as RegisterPrimOp() and Value::mkPrimOp().
+     */
+    void check();
 };
 
 /**
@@ -827,7 +839,7 @@ std::string showType(const Value & v);
 /**
  * If `path` refers to a directory, then append "/default.nix".
  */
-SourcePath resolveExprPath(const SourcePath & path);
+SourcePath resolveExprPath(SourcePath path);
 
 struct InvalidPathError : EvalError
 {

src/libexpr/flake/config.cc

@@ -1,6 +1,7 @@
-#include "flake.hh"
+#include "users.hh"
 #include "globals.hh"
 #include "fetch-settings.hh"
+#include "flake.hh"
 
 #include <nlohmann/json.hpp>
 

src/libexpr/flake/flake.cc

@@ -1,3 +1,4 @@
+#include "terminal.hh"
 #include "flake.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
@@ -8,6 +9,7 @@
 #include "fetchers.hh"
 #include "finally.hh"
 #include "fetch-settings.hh"
+#include "value-to-json.hh"
 
 namespace nix {
 
@@ -140,8 +142,13 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer);
                         break;
                     default:
-                        throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
+                        if (attr.name == state.symbols.create("publicKeys")) {
-                            state.symbols[attr.name], showType(*attr.value));
+                            experimentalFeatureSettings.require(Xp::VerifiedFetches);
+                            NixStringContext emptyContext = {};
+                            attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump());
+                        } else
+                            throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
+                                state.symbols[attr.name], showType(*attr.value));
                 }
                 #pragma GCC diagnostic pop
             }
@@ -447,8 +454,8 @@ LockedFlake lockFlake(
 
                     assert(input.ref);
 
-                    /* Do we have an entry in the existing lock file? And we
+                    /* Do we have an entry in the existing lock file?
-                       don't have a --update-input flag for this input? */
+                       And the input is not in updateInputs? */
                     std::shared_ptr<LockedNode> oldLock;
 
                     updatesUsed.insert(inputPath);
@@ -472,9 +479,8 @@ LockedFlake lockFlake(
 
                         node->inputs.insert_or_assign(id, childNode);
 
-                        /* If we have an --update-input flag for an input
+                        /* If we have this input in updateInputs, then we
-                           of this input, then we must fetch the flake to
+                           must fetch the flake to update it. */
-                           update it. */
                         auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
                         auto mustRefetch =
@@ -616,19 +622,14 @@ LockedFlake lockFlake(
 
         for (auto & i : lockFlags.inputUpdates)
             if (!updatesUsed.count(i))
-                warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+                warn("'%s' does not match any input of this flake", printInputPath(i));
 
         /* Check 'follows' inputs. */
         newLockFile.check();
 
         debug("new lock file: %s", newLockFile);
 
-        auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
         auto sourcePath = topRef.input.getSourcePath();
-        auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
-        if (lockFlags.outputLockFilePath) {
-            outputLockFilePath = lockFlags.outputLockFilePath;
-        }
 
         /* Check whether we need to / can write the new lock file. */
         if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) {
@@ -636,7 +637,7 @@ LockedFlake lockFlake(
             auto diff = LockFile::diff(oldLockFile, newLockFile);
 
             if (lockFlags.writeLockFile) {
-                if (outputLockFilePath) {
+                if (sourcePath || lockFlags.outputLockFilePath) {
                     if (auto unlockedInput = newLockFile.isUnlocked()) {
                         if (fetchSettings.warnDirty)
                             warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
@@ -644,41 +645,48 @@ LockedFlake lockFlake(
                         if (!lockFlags.updateLockFile)
                             throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
 
-                        bool lockFileExists = pathExists(*outputLockFilePath);
+                        auto newLockFileS = fmt("%s\n", newLockFile);
+
+                        if (lockFlags.outputLockFilePath) {
+                            if (lockFlags.commitLockFile)
+                                throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible");
+                            writeFile(*lockFlags.outputLockFilePath, newLockFileS);
+                        } else {
+                            auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+                            auto outputLockFilePath = *sourcePath + "/" + relPath;
+
+                            bool lockFileExists = pathExists(outputLockFilePath);
 
-                        if (lockFileExists) {
                             auto s = chomp(diff);
-                            if (s.empty())
+                            if (lockFileExists) {
-                                warn("updating lock file '%s'", *outputLockFilePath);
+                                if (s.empty())
-                            else
+                                    warn("updating lock file '%s'", outputLockFilePath);
-                                warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
+                                else
-                        } else
+                                    warn("updating lock file '%s':\n%s", outputLockFilePath, s);
-                            warn("creating lock file '%s'", *outputLockFilePath);
+                            } else
+                                warn("creating lock file '%s': \n%s", outputLockFilePath, s);
 
-                        newLockFile.write(*outputLockFilePath);
+                            std::optional<std::string> commitMessage = std::nullopt;
 
-                        std::optional<std::string> commitMessage = std::nullopt;
+                            if (lockFlags.commitLockFile) {
-                        if (lockFlags.commitLockFile) {
+                                std::string cm;
-                            if (lockFlags.outputLockFilePath) {
-                                throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
-                            }
-                            std::string cm;
 
-                            cm = fetchSettings.commitLockFileSummary.get();
+                                cm = fetchSettings.commitLockFileSummary.get();
 
-                            if (cm == "") {
+                                if (cm == "") {
-                                cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
+                                    cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
+                                }
+
+                                cm += "\n\nFlake lock file updates:\n\n";
+                                cm += filterANSIEscapes(diff, true);
+                                commitMessage = cm;
                             }
 
-                            cm += "\n\nFlake lock file updates:\n\n";
+                            topRef.input.putFile(
-                            cm += filterANSIEscapes(diff, true);
+                                CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
-                            commitMessage = cm;
+                                newLockFileS, commitMessage);
                         }
 
-                        topRef.input.markChangedFile(
-                            (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
-                            commitMessage);
-
                         /* Rewriting the lockfile changed the top-level
                            repo, so we should re-read it. FIXME: we could
                            also just clear the 'rev' field... */

src/libexpr/flake/lockfile.cc

@@ -214,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
     return stream;
 }
 
-void LockFile::write(const Path & path) const
-{
-    createDirs(dirOf(path));
-    writeFile(path, fmt("%s\n", *this));
-}
-
 std::optional<FlakeRef> LockFile::isUnlocked() const
 {
     std::set<ref<const Node>> nodes;

src/libexpr/flake/lockfile.hh

@@ -65,8 +65,6 @@ struct LockFile
 
     static LockFile read(const Path & path);
 
-    void write(const Path & path) const;
-
     /**
      * Check whether this lock file has any unlocked inputs.
      */

src/libexpr/get-drvs.cc

@@ -1,5 +1,4 @@
 #include "get-drvs.hh"
-#include "util.hh"
 #include "eval-inline.hh"
 #include "derivations.hh"
 #include "store-api.hh"

src/libexpr/local.mk

@@ -43,7 +43,9 @@ $(foreach i, $(wildcard src/libexpr/value/*.hh), \
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
+
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
 
 $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
 

src/libexpr/parser.y

@@ -19,6 +19,7 @@
 #include <variant>
 
 #include "util.hh"
+#include "users.hh"
 
 #include "nixexpr.hh"
 #include "eval.hh"
@@ -685,17 +686,25 @@ Expr * EvalState::parse(
 }
 
 
-SourcePath resolveExprPath(const SourcePath & path)
+SourcePath resolveExprPath(SourcePath path)
 {
+    unsigned int followCount = 0, maxFollow = 1024;
+
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
-    auto path2 = path.resolveSymlinks();
+    while (true) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
+        if (path.lstat().type != InputAccessor::tSymlink) break;
+        path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))};
+    }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (path2.lstat().type == InputAccessor::tDirectory)
+    if (path.lstat().type == InputAccessor::tDirectory)
-        return path2 + "default.nix";
+        return path + "default.nix";
 
-    return path2;
+    return path;
 }
 
 

src/libexpr/primops.cc

@@ -10,6 +10,7 @@
 #include "path-references.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "processes.hh"
 #include "value-to-json.hh"
 #include "value-to-xml.hh"
 #include "primops.hh"
@@ -28,7 +29,6 @@
 
 #include <cmath>
 
-
 namespace nix {
 
 
@@ -824,7 +824,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
         auto message = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
                 false, false).toOwned();
-        e.addTrace(nullptr, message, true);
+        e.addTrace(nullptr, hintfmt(message), true);
         throw;
     }
 }
@@ -1548,10 +1548,8 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
 
     try {
         auto checked = state.checkSourcePath(path);
-        auto exists = checked.pathExists();
+        auto st = checked.maybeLstat();
-        if (exists && mustBeDir) {
+        auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
-            exists = checked.lstat().type == InputAccessor::tDirectory;
-        }
         v.mkBool(exists);
     } catch (SysError & e) {
         /* Don't give away info from errors while canonicalising
@@ -2551,6 +2549,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args
     /* Get the attribute names to be removed.
        We keep them as Attrs instead of Symbols so std::set_difference
        can be used to remove them from attrs[0]. */
+    // 64: large enough to fit the attributes of a derivation
     boost::container::small_vector<Attr, 64> names;
     names.reserve(args[1]->listSize());
     for (auto elem : args[1]->listItems()) {
@@ -2730,8 +2729,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    Value * res[args[1]->listSize()];
+    boost::container::small_vector<Value *, nonRecursiveStackReservation> res(args[1]->listSize());
-    unsigned int found = 0;
+    size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
         state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs");
@@ -3065,9 +3064,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    // FIXME: putting this on the stack is risky.
+    boost::container::small_vector<Value *, nonRecursiveStackReservation> vs(args[1]->listSize());
-    Value * vs[args[1]->listSize()];
+    size_t k = 0;
-    unsigned int k = 0;
 
     bool same = true;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
@@ -3192,10 +3190,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar
     state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all"));
     state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all"));
 
+    std::string_view errorCtx = any
+        ? "while evaluating the return value of the function passed to builtins.any"
+        : "while evaluating the return value of the function passed to builtins.all";
+
     Value vTmp;
     for (auto elem : args[1]->listItems()) {
         state.callFunction(*args[0], *elem, vTmp, pos);
-        bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all"));
+        bool res = state.forceBool(vTmp, pos, errorCtx);
         if (res == any) {
             v.mkBool(any);
             return;
@@ -3451,7 +3453,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    Value lists[nrLists];
+    boost::container::small_vector<Value, conservativeStackReservation> lists(nrLists);
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {

src/libexpr/primops.hh

@@ -8,6 +8,22 @@
 
 namespace nix {
 
+/**
+ * For functions where we do not expect deep recursion, we can use a sizable
+ * part of the stack a free allocation space.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t nonRecursiveStackReservation = 128;
+
+/**
+ * Functions that maybe applied to self-similar inputs, such as concatMap on a
+ * tree, should reserve a smaller part of the stack for allocation.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t conservativeStackReservation = 16;
+
 struct RegisterPrimOp
 {
     typedef std::vector<PrimOp> PrimOps;

src/libexpr/primops/fetchTree.cc

@@ -7,6 +7,7 @@
 #include "registry.hh"
 #include "tarball.hh"
 #include "url.hh"
+#include "value-to-json.hh"
 
 #include <ctime>
 #include <iomanip>
@@ -125,6 +126,10 @@ static void fetchTree(
                 attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean});
             else if (attr.value->type() == nInt)
                 attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer));
+            else if (state.symbols[attr.name] == "publicKeys") {
+                experimentalFeatureSettings.require(Xp::VerifiedFetches);
+                attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
+            }
             else
                 state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                     state.symbols[attr.name], showType(*attr.value)));
@@ -223,6 +228,7 @@ static RegisterPrimOp primop_fetchTree({
           ```
     )",
     .fun = prim_fetchTree,
+    .experimentalFeature = Xp::FetchTree,
 });
 
 static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v,
@@ -427,6 +433,42 @@ static RegisterPrimOp primop_fetchGit({
         With this argument being true, it's possible to load a `rev` from *any* `ref`
         (by default only `rev`s from the specified `ref` are supported).
 
+      - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
+
+        Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.
+        If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes.
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+      - `publicKey`
+
+        The public key against which `rev` is verified if `verifyCommit` is enabled.
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+      - `keytype` (default: `"ssh-ed25519"`)
+
+        The key type of `publicKey`.
+        Possible values:
+        - `"ssh-dsa"`
+        - `"ssh-ecdsa"`
+        - `"ssh-ecdsa-sk"`
+        - `"ssh-ed25519"`
+        - `"ssh-ed25519-sk"`
+        - `"ssh-rsa"`
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+      - `publicKeys`
+
+        The public keys against which `rev` is verified if `verifyCommit` is enabled.
+        Must be given as a list of attribute sets with the following form:
+        ```nix
+        {
+          key = "<public key>";
+          type = "<key type>"; # optional, default: "ssh-ed25519"
+        }
+        ```
+        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+
       Here are some examples of how to use `fetchGit`.
 
         - To fetch a private repository over SSH:
@@ -501,6 +543,21 @@ static RegisterPrimOp primop_fetchGit({
           }
           ```
 
+        - To verify the commit signature:
+
+          ```nix
+          builtins.fetchGit {
+            url = "ssh://git@github.com/nixos/nix.git";
+            verifyCommit = true;
+            publicKeys = [
+                {
+                  type = "ssh-ed25519";
+                  key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi";
+                }
+            ];
+          }
+          ```
+
           Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting.
 
           This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).

src/libexpr/search-path.cc

@@ -1,5 +1,4 @@
 #include "search-path.hh"
-#include "util.hh"
 
 namespace nix {
 

src/libexpr/tests/local.mk

@@ -6,7 +6,11 @@ libexpr-tests_NAME := libnixexpr-tests
 
 libexpr-tests_DIR := $(d)
 
-libexpr-tests_INSTALL_DIR :=
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libexpr-tests_INSTALL_DIR := $(checkbindir)
+else
+  libexpr-tests_INSTALL_DIR :=
+endif
 
 libexpr-tests_SOURCES := \
     $(wildcard $(d)/*.cc) \

src/libexpr/tests/value/print.cc

@@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda)
 TEST_F(ValuePrintingTests, vPrimOp)
 {
     Value vPrimOp;
-    vPrimOp.mkPrimOp(nullptr);
+    PrimOp primOp{};
+    vPrimOp.mkPrimOp(&primOp);
 
     test(vPrimOp, "<PRIMOP>");
 }

src/libexpr/value-to-json.cc

@@ -1,7 +1,7 @@
 #include "value-to-json.hh"
 #include "eval-inline.hh"
-#include "util.hh"
 #include "store-api.hh"
+#include "signals.hh"
 
 #include <cstdlib>
 #include <iomanip>

src/libexpr/value-to-xml.cc

@@ -1,7 +1,7 @@
 #include "value-to-xml.hh"
 #include "xml-writer.hh"
 #include "eval-inline.hh"
-#include "util.hh"
+#include "signals.hh"
 
 #include <cstdlib>
 

src/libexpr/value.hh

@@ -354,13 +354,7 @@ public:
         // Value will be overridden anyways
     }
 
-    inline void mkPrimOp(PrimOp * p)
+    void mkPrimOp(PrimOp * p);
-    {
-        clearValue();
-        internalType = tPrimOp;
-        primOp = p;
-    }
-
 
     inline void mkPrimOpApp(Value * l, Value * r)
     {

src/libexpr/value/context.cc

@@ -1,3 +1,4 @@
+#include "util.hh"
 #include "value/context.hh"
 
 #include <optional>

src/libexpr/value/context.hh

@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "util.hh"
 #include "comparator.hh"
 #include "derived-path.hh"
 #include "variant-wrapper.hh"

src/libfetchers/cache.cc

@@ -1,4 +1,5 @@
 #include "cache.hh"
+#include "users.hh"
 #include "sqlite.hh"
 #include "sync.hh"
 #include "store-api.hh"

src/libfetchers/cache.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "fetchers.hh"
+#include "path.hh"
 
 namespace nix::fetchers {
 

src/libfetchers/fetch-settings.hh

@@ -3,7 +3,6 @@
 
 #include "types.hh"
 #include "config.hh"
-#include "util.hh"
 
 #include <map>
 #include <limits>

src/libfetchers/fetchers.cc

@@ -5,12 +5,31 @@
 
 namespace nix::fetchers {
 
-std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
+using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
+
+std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
 
 void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
 {
-    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
+    if (!inputSchemes)
-    inputSchemes->push_back(std::move(inputScheme));
+        inputSchemes = std::make_unique<InputSchemeMap>();
+    auto schemeName = inputScheme->schemeName();
+    if (inputSchemes->count(schemeName) > 0)
+        throw Error("Input scheme with name %s already registered", schemeName);
+    inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
+}
+
+nlohmann::json dumpRegisterInputSchemeInfo() {
+    using nlohmann::json;
+
+    auto res = json::object();
+
+    for (auto & [name, scheme] : *inputSchemes) {
+        auto & r = res[name] = json::object();
+        r["allowedAttrs"] = scheme->allowedAttrs();
+    }
+
+    return res;
 }
 
 Input Input::fromURL(const std::string & url, bool requireTree)
@@ -33,7 +52,7 @@ static void fixupInput(Input & input)
 
 Input Input::fromURL(const ParsedURL & url, bool requireTree)
 {
-    for (auto & inputScheme : *inputSchemes) {
+    for (auto & [_, inputScheme] : *inputSchemes) {
         auto res = inputScheme->inputFromURL(url, requireTree);
         if (res) {
             experimentalFeatureSettings.require(inputScheme->experimentalFeature());
@@ -48,20 +67,44 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree)
 
 Input Input::fromAttrs(Attrs && attrs)
 {
-    for (auto & inputScheme : *inputSchemes) {
+    auto schemeName = ({
-        auto res = inputScheme->inputFromAttrs(attrs);
+        auto schemeNameOpt = maybeGetStrAttr(attrs, "type");
-        if (res) {
+        if (!schemeNameOpt)
-            experimentalFeatureSettings.require(inputScheme->experimentalFeature());
+            throw Error("'type' attribute to specify input scheme is required but not provided");
-            res->scheme = inputScheme;
+        *std::move(schemeNameOpt);
-            fixupInput(*res);
+    });
-            return std::move(*res);
-        }
-    }
 
-    Input input;
+    auto raw = [&]() {
-    input.attrs = attrs;
+        // Return an input without a scheme; most operations will fail,
-    fixupInput(input);
+        // but not all of them. Doing this is to support those other
-    return input;
+        // operations which are supposed to be robust on
+        // unknown/uninterpretable inputs.
+        Input input;
+        input.attrs = attrs;
+        fixupInput(input);
+        return input;
+    };
+
+    std::shared_ptr<InputScheme> inputScheme = ({
+        auto i = inputSchemes->find(schemeName);
+        i == inputSchemes->end() ? nullptr : i->second;
+    });
+
+    if (!inputScheme) return raw();
+
+    experimentalFeatureSettings.require(inputScheme->experimentalFeature());
+
+    auto allowedAttrs = inputScheme->allowedAttrs();
+
+    for (auto & [name, _] : attrs)
+        if (name != "type" && allowedAttrs.count(name) == 0)
+            throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
+
+    auto res = inputScheme->inputFromAttrs(attrs);
+    if (!res) return raw();
+    res->scheme = inputScheme;
+    fixupInput(*res);
+    return std::move(*res);
 }
 
 ParsedURL Input::toURL() const
@@ -196,12 +239,13 @@ std::optional<Path> Input::getSourcePath() const
     return scheme->getSourcePath(*this);
 }
 
-void Input::markChangedFile(
+void Input::putFile(
-    std::string_view file,
+    const CanonPath & path,
+    std::string_view contents,
     std::optional<std::string> commitMsg) const
 {
     assert(scheme);
-    return scheme->markChangedFile(*this, file, commitMsg);
+    return scheme->putFile(*this, path, contents, commitMsg);
 }
 
 std::string Input::getName() const
@@ -292,14 +336,18 @@ Input InputScheme::applyOverrides(
     return input;
 }
 
-std::optional<Path> InputScheme::getSourcePath(const Input & input)
+std::optional<Path> InputScheme::getSourcePath(const Input & input) const
 {
     return {};
 }
 
-void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
+void InputScheme::putFile(
+    const Input & input,
+    const CanonPath & path,
+    std::string_view contents,
+    std::optional<std::string> commitMsg) const
 {
-    assert(false);
+    throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
 }
 
 void InputScheme::clone(const Input & input, const Path & destDir) const
@@ -307,9 +355,14 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
     throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
-std::optional<ExperimentalFeature> InputScheme::experimentalFeature()
+std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
 {
     return {};
 }
 
+std::string publicKeys_to_string(const std::vector<PublicKey>& publicKeys)
+{
+    return ((nlohmann::json) publicKeys).dump();
+}
+
 }

src/libfetchers/fetchers.hh

@@ -3,13 +3,14 @@
 
 #include "types.hh"
 #include "hash.hh"
-#include "path.hh"
+#include "canon-path.hh"
 #include "attrs.hh"
 #include "url.hh"
 
 #include <memory>
+#include <nlohmann/json_fwd.hpp>
 
-namespace nix { class Store; }
+namespace nix { class Store; class StorePath; }
 
 namespace nix::fetchers {
 
@@ -90,8 +91,13 @@ public:
 
     std::optional<Path> getSourcePath() const;
 
-    void markChangedFile(
+    /**
-        std::string_view file,
+     * Write a file to this input, for input types that support
+     * writing. Optionally commit the change (for e.g. Git inputs).
+     */
+    void putFile(
+        const CanonPath & path,
+        std::string_view contents,
         std::optional<std::string> commitMsg) const;
 
     std::string getName() const;
@@ -126,6 +132,24 @@ struct InputScheme
 
     virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;
 
+    /**
+     * What is the name of the scheme?
+     *
+     * The `type` attribute is used to select which input scheme is
+     * used, and then the other fields are forwarded to that input
+     * scheme.
+     */
+    virtual std::string_view schemeName() const = 0;
+
+    /**
+     * Allowed attributes in an attribute set that is converted to an
+     * input.
+     *
+     * `type` is not included from this set, because the `type` field is
+      parsed first to choose which scheme; `type` is always required.
+     */
+    virtual StringSet allowedAttrs() const = 0;
+
     virtual ParsedURL toURL(const Input & input) const;
 
     virtual Input applyOverrides(
@@ -135,16 +159,20 @@ struct InputScheme
 
     virtual void clone(const Input & input, const Path & destDir) const;
 
-    virtual std::optional<Path> getSourcePath(const Input & input);
+    virtual std::optional<Path> getSourcePath(const Input & input) const;
 
-    virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
+    virtual void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const;
 
     virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
 
     /**
      * Is this `InputScheme` part of an experimental feature?
      */
-    virtual std::optional<ExperimentalFeature> experimentalFeature();
+    virtual std::optional<ExperimentalFeature> experimentalFeature() const;
 
     virtual bool isDirect(const Input & input) const
     { return true; }
@@ -152,4 +180,15 @@ struct InputScheme
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
+nlohmann::json dumpRegisterInputSchemeInfo();
+
+struct PublicKey
+{
+    std::string type = "ssh-ed25519";
+    std::string key;
+};
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key)
+
+std::string publicKeys_to_string(const std::vector<PublicKey>&);
+
 }

src/libfetchers/fs-input-accessor.cc

@@ -36,11 +36,11 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
     }
 
-    Stat lstat(const CanonPath & path) override
+    std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
         auto absPath = makeAbsPath(path);
         checkAllowed(absPath);
-        return PosixSourceAccessor::lstat(absPath);
+        return PosixSourceAccessor::maybeLstat(absPath);
     }
 
     DirEntries readDirectory(const CanonPath & path) override

src/libfetchers/git.cc

@@ -1,11 +1,12 @@
 #include "fetchers.hh"
+#include "users.hh"
 #include "cache.hh"
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
 #include "pathlocks.hh"
-#include "util.hh"
+#include "processes.hh"
 #include "git.hh"
 
 #include "fetch-settings.hh"
@@ -143,6 +144,69 @@ struct WorkdirInfo
     bool hasHead = false;
 };
 
+std::vector<PublicKey> getPublicKeys(const Attrs & attrs) {
+    std::vector<PublicKey> publicKeys;
+    if (attrs.contains("publicKeys")) {
+        nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys"));
+        ensureType(publicKeysJson, nlohmann::json::value_t::array);
+        publicKeys = publicKeysJson.get<std::vector<PublicKey>>();
+    }
+    else {
+        publicKeys = {};
+    }
+    if (attrs.contains("publicKey"))
+        publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")});
+    return publicKeys;
+}
+
+void doCommitVerification(const Path repoDir, const Path gitDir, const std::string rev, const std::vector<PublicKey>& publicKeys) {
+    // Create ad-hoc allowedSignersFile and populate it with publicKeys
+    auto allowedSignersFile = createTempFile().second;
+    std::string allowedSigners;
+    for (const PublicKey& k : publicKeys) {
+        if (k.type != "ssh-dsa"
+            && k.type != "ssh-ecdsa"
+            && k.type != "ssh-ecdsa-sk"
+            && k.type != "ssh-ed25519"
+            && k.type != "ssh-ed25519-sk"
+            && k.type != "ssh-rsa")
+            warn("Unknown keytype: %s\n"
+                 "Please use one of\n"
+                 "- ssh-dsa\n"
+                 "  ssh-ecdsa\n"
+                 "  ssh-ecdsa-sk\n"
+                 "  ssh-ed25519\n"
+                 "  ssh-ed25519-sk\n"
+                 "  ssh-rsa", k.type);
+        allowedSigners += "* " + k.type + " " + k.key + "\n";
+    }
+    writeFile(allowedSignersFile, allowedSigners);
+
+    // Run verification command
+    auto [status, output] = runProgram(RunOptions {
+            .program = "git",
+            .args = {"-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, "-C", repoDir,
+                     "--git-dir", gitDir, "verify-commit", rev},
+            .mergeStderrToStdout = true,
+    });
+
+    /* Evaluate result through status code and checking if public key fingerprints appear on stderr
+     * This is neccessary because the git command might also succeed due to the commit being signed by gpg keys
+     * that are present in the users key agent. */
+    std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
+    for (const PublicKey& k : publicKeys){
+        // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
+        auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
+        auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
+        re += "(" + escaped_fingerprint + ")";
+    }
+    re += "]";
+    if (status == 0 && std::regex_search(output, std::regex(re)))
+        printTalkative("Signature verification on commit %s succeeded", rev);
+    else
+        throw Error("Commit signature verification on commit %s failed: \n%s", rev, output);
+}
+
 // Returns whether a git workdir is clean and has commits.
 WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
 {
@@ -272,9 +336,9 @@ struct GitInputScheme : InputScheme
         attrs.emplace("type", "git");
 
         for (auto & [name, value] : url.query) {
-            if (name == "rev" || name == "ref")
+            if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
                 attrs.emplace(name, value);
-            else if (name == "shallow" || name == "submodules" || name == "allRefs")
+            else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -285,17 +349,47 @@ struct GitInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
+
+    std::string_view schemeName() const override
+    {
+        return "git";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "url",
+            "ref",
+            "rev",
+            "shallow",
+            "submodules",
+            "lastModified",
+            "revCount",
+            "narHash",
+            "allRefs",
+            "name",
+            "dirtyRev",
+            "dirtyShortRev",
+            "verifyCommit",
+            "keytype",
+            "publicKey",
+            "publicKeys",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "git") return {};
+        for (auto & [name, _] : attrs)
-
+            if (name == "verifyCommit"
-        for (auto & [name, value] : attrs)
+                || name == "keytype"
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
+                || name == "publicKey"
-                throw Error("unsupported Git input attribute '%s'", name);
+                || name == "publicKeys")
+                experimentalFeatureSettings.require(Xp::VerifiedFetches);
 
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
         maybeGetBoolAttr(attrs, "allRefs");
+        maybeGetBoolAttr(attrs, "verifyCommit");
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
             if (std::regex_search(*ref, badGitRefRegex))
@@ -318,6 +412,15 @@ struct GitInputScheme : InputScheme
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
         if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
             url.query.insert_or_assign("shallow", "1");
+        if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
+            url.query.insert_or_assign("verifyCommit", "1");
+        auto publicKeys = getPublicKeys(input.attrs);
+        if (publicKeys.size() == 1) {
+            url.query.insert_or_assign("keytype", publicKeys.at(0).type);
+            url.query.insert_or_assign("publicKey", publicKeys.at(0).key);
+        }
+        else if (publicKeys.size() > 1)
+            url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys));
         return url;
     }
 
@@ -354,7 +457,7 @@ struct GitInputScheme : InputScheme
         runProgram("git", true, args, {}, true);
     }
 
-    std::optional<Path> getSourcePath(const Input & input) override
+    std::optional<Path> getSourcePath(const Input & input) const override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         if (url.scheme == "file" && !input.getRef() && !input.getRev())
@@ -362,18 +465,26 @@ struct GitInputScheme : InputScheme
         return {};
     }
 
-    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const override
     {
-        auto sourcePath = getSourcePath(input);
+        auto root = getSourcePath(input);
-        assert(sourcePath);
+        if (!root)
+            throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
+
+        writeFile((CanonPath(*root) + path).abs(), contents);
+
         auto gitDir = ".git";
 
         runProgram("git", true,
-            { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) });
+            { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
         if (commitMsg)
             runProgram("git", true,
-                { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg });
+                { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
     }
 
     std::pair<bool, std::string> getActualUrl(const Input & input) const
@@ -399,6 +510,8 @@ struct GitInputScheme : InputScheme
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
         bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
+        std::vector<PublicKey> publicKeys = getPublicKeys(input.attrs);
+        bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty());
 
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
@@ -419,6 +532,8 @@ struct GitInputScheme : InputScheme
                 {"type", cacheType},
                 {"name", name},
                 {"rev", input.getRev()->gitRev()},
+                {"verifyCommit", verifyCommit},
+                {"publicKeys", publicKeys_to_string(publicKeys)},
             });
         };
 
@@ -441,12 +556,15 @@ struct GitInputScheme : InputScheme
         auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
-        /* If this is a local directory and no ref or revision is given,
+        /* If this is a local directory, no ref or revision is given and no signature verification is needed,
            allow fetching directly from a dirty workdir. */
         if (!input.getRef() && !input.getRev() && isLocal) {
             auto workdirInfo = getWorkdirInfo(input, actualUrl);
             if (!workdirInfo.clean) {
-                return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
+                if (verifyCommit)
+                    throw Error("Can't fetch from a dirty workdir with commit signature verification enabled.");
+                else
+                    return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
             }
         }
 
@@ -454,6 +572,8 @@ struct GitInputScheme : InputScheme
             {"type", cacheType},
             {"name", name},
             {"url", actualUrl},
+            {"verifyCommit", verifyCommit},
+            {"publicKeys", publicKeys_to_string(publicKeys)},
         });
 
         Path repoDir;
@@ -611,6 +731,9 @@ struct GitInputScheme : InputScheme
             );
         }
 
+        if (verifyCommit)
+            doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys);
+
         if (submodules) {
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);

src/libfetchers/github.cc

@@ -27,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
 struct GitArchiveInputScheme : InputScheme
 {
-    virtual std::string type() const = 0;
-
     virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
     {
-        if (url.scheme != type()) return {};
+        if (url.scheme != schemeName()) return {};
 
         auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
 
@@ -91,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme
             throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
 
         Input input;
-        input.attrs.insert_or_assign("type", type());
+        input.attrs.insert_or_assign("type", std::string { schemeName() });
         input.attrs.insert_or_assign("owner", path[0]);
         input.attrs.insert_or_assign("repo", path[1]);
         if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
@@ -101,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "owner",
+            "repo",
+            "ref",
+            "rev",
+            "narHash",
+            "lastModified",
+            "host",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != type()) return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
-                throw Error("unsupported input attribute '%s'", name);
-
         getStrAttr(attrs, "owner");
         getStrAttr(attrs, "repo");
 
@@ -128,7 +133,7 @@ struct GitArchiveInputScheme : InputScheme
         if (ref) path += "/" + *ref;
         if (rev) path += "/" + rev->to_string(HashFormat::Base16, false);
         return ParsedURL {
-            .scheme = type(),
+            .scheme = std::string { schemeName() },
             .path = path,
         };
     }
@@ -220,7 +225,7 @@ struct GitArchiveInputScheme : InputScheme
         return {result.storePath, input};
     }
 
-    std::optional<ExperimentalFeature> experimentalFeature() override
+    std::optional<ExperimentalFeature> experimentalFeature() const override
     {
         return Xp::Flakes;
     }
@@ -228,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme
 
 struct GitHubInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "github"; }
+    std::string_view schemeName() const override { return "github"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -309,7 +314,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
 struct GitLabInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "gitlab"; }
+    std::string_view schemeName() const override { return "gitlab"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
@@ -377,7 +382,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
 struct SourceHutInputScheme : GitArchiveInputScheme
 {
-    std::string type() const override { return "sourcehut"; }
+    std::string_view schemeName() const override { return "sourcehut"; }
 
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {

src/libfetchers/indirect.cc

@@ -1,5 +1,6 @@
 #include "fetchers.hh"
 #include "url-parts.hh"
+#include "path.hh"
 
 namespace nix::fetchers {
 
@@ -49,14 +50,23 @@ struct IndirectInputScheme : InputScheme
         return input;
     }
 
+    std::string_view schemeName() const override
+    {
+        return "indirect";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "id",
+            "ref",
+            "rev",
+            "narHash",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
-                throw Error("unsupported indirect input attribute '%s'", name);
-
         auto id = getStrAttr(attrs, "id");
         if (!std::regex_match(id, flakeRegex))
             throw BadURL("'%s' is not a valid flake ID", id);
@@ -92,7 +102,7 @@ struct IndirectInputScheme : InputScheme
         throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
     }
 
-    std::optional<ExperimentalFeature> experimentalFeature() override
+    std::optional<ExperimentalFeature> experimentalFeature() const override
     {
         return Xp::Flakes;
     }

src/libfetchers/input-accessor.hh

@@ -1,8 +1,10 @@
 #pragma once
+///@file
 
 #include "source-accessor.hh"
 #include "ref.hh"
 #include "types.hh"
+#include "file-system.hh"
 #include "repair-flag.hh"
 #include "content-address.hh"
 
@@ -14,7 +16,7 @@ struct SourcePath;
 class StorePath;
 class Store;
 
-struct InputAccessor : SourceAccessor, std::enable_shared_from_this<InputAccessor>
+struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
 {
     /**
      * Return the maximum last-modified time of the files in this

src/libfetchers/memory-input-accessor.cc

@@ -1,48 +1,16 @@
 #include "memory-input-accessor.hh"
+#include "memory-source-accessor.hh"
 
 namespace nix {
 
-struct MemoryInputAccessorImpl : MemoryInputAccessor
+struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor
 {
-    std::map<CanonPath, std::string> files;
-
-    std::string readFile(const CanonPath & path) override
-    {
-        auto i = files.find(path);
-        if (i == files.end())
-            throw Error("file '%s' does not exist", path);
-        return i->second;
-    }
-
-    bool pathExists(const CanonPath & path) override
-    {
-        auto i = files.find(path);
-        return i != files.end();
-    }
-
-    Stat lstat(const CanonPath & path) override
-    {
-        auto i = files.find(path);
-        if (i != files.end())
-            return Stat { .type = tRegular, .isExecutable = false };
-        throw Error("file '%s' does not exist", path);
-    }
-
-    DirEntries readDirectory(const CanonPath & path) override
-    {
-        return {};
-    }
-
-    std::string readLink(const CanonPath & path) override
-    {
-        throw UnimplementedError("MemoryInputAccessor::readLink");
-    }
-
     SourcePath addFile(CanonPath path, std::string && contents) override
     {
-        files.emplace(path, std::move(contents));
+        return {
-
+            ref(shared_from_this()),
-        return {ref(shared_from_this()), std::move(path)};
+            MemorySourceAccessor::addFile(path, std::move(contents))
+        };
     }
 };
 

src/libfetchers/mercurial.cc

@@ -1,4 +1,6 @@
 #include "fetchers.hh"
+#include "processes.hh"
+#include "users.hh"
 #include "cache.hh"
 #include "globals.hh"
 #include "tarfile.hh"
@@ -69,14 +71,25 @@ struct MercurialInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
+    std::string_view schemeName() const override
+    {
+        return "hg";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "url",
+            "ref",
+            "rev",
+            "revCount",
+            "narHash",
+            "name",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
-                throw Error("unsupported Mercurial input attribute '%s'", name);
-
         parseURL(getStrAttr(attrs, "url"));
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@@ -109,7 +122,7 @@ struct MercurialInputScheme : InputScheme
         return res;
     }
 
-    std::optional<Path> getSourcePath(const Input & input) override
+    std::optional<Path> getSourcePath(const Input & input) const override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         if (url.scheme == "file" && !input.getRef() && !input.getRev())
@@ -117,18 +130,27 @@ struct MercurialInputScheme : InputScheme
         return {};
     }
 
-    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const override
     {
-        auto sourcePath = getSourcePath(input);
+        auto [isLocal, repoPath] = getActualUrl(input);
-        assert(sourcePath);
+        if (!isLocal)
+            throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string());
+
+        auto absPath = CanonPath(repoPath) + path;
+
+        writeFile(absPath.abs(), contents);
 
         // FIXME: shut up if file is already tracked.
         runHg(
-            { "add", *sourcePath + "/" + std::string(file) });
+            { "add", absPath.abs() });
 
         if (commitMsg)
             runHg(
-                { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
+                { "commit", absPath.abs(), "-m", *commitMsg });
     }
 
     std::pair<bool, std::string> getActualUrl(const Input & input) const

src/libfetchers/path.cc

@@ -32,23 +32,30 @@ struct PathInputScheme : InputScheme
         return input;
     }
 
+    std::string_view schemeName() const override
+    {
+        return "path";
+    }
+
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "path",
+            /* Allow the user to pass in "fake" tree info
+               attributes. This is useful for making a pinned tree work
+               the same as the repository from which is exported (e.g.
+               path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
+             */
+            "rev",
+            "revCount",
+            "lastModified",
+            "narHash",
+        };
+    }
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
         getStrAttr(attrs, "path");
 
-        for (auto & [name, value] : attrs)
-            /* Allow the user to pass in "fake" tree info
-               attributes. This is useful for making a pinned tree
-               work the same as the repository from which is exported
-               (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
-            if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
-                // checked in Input::fromAttrs
-                ;
-            else
-                throw Error("unsupported path input attribute '%s'", name);
-
         Input input;
         input.attrs = attrs;
         return input;
@@ -66,14 +73,28 @@ struct PathInputScheme : InputScheme
         };
     }
 
-    std::optional<Path> getSourcePath(const Input & input) override
+    std::optional<Path> getSourcePath(const Input & input) const override
     {
         return getStrAttr(input.attrs, "path");
     }
 
-    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    void putFile(
+        const Input & input,
+        const CanonPath & path,
+        std::string_view contents,
+        std::optional<std::string> commitMsg) const override
     {
-        // nothing to do
+        writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents);
+    }
+
+    CanonPath getAbsPath(const Input & input) const
+    {
+        auto path = getStrAttr(input.attrs, "path");
+
+        if (path[0] == '/')
+            return CanonPath(path);
+
+        throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
     }
 
     std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
@@ -121,7 +142,7 @@ struct PathInputScheme : InputScheme
         return {std::move(*storePath), input};
     }
 
-    std::optional<ExperimentalFeature> experimentalFeature() override
+    std::optional<ExperimentalFeature> experimentalFeature() const override
     {
         return Xp::Flakes;
     }

src/libfetchers/registry.cc

@@ -1,6 +1,6 @@
 #include "registry.hh"
 #include "tarball.hh"
-#include "util.hh"
+#include "users.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"

src/libfetchers/tarball.cc

@@ -184,7 +184,6 @@ DownloadTarballResult downloadTarball(
 // An input scheme corresponding to a curl-downloadable resource.
 struct CurlInputScheme : InputScheme
 {
-    virtual const std::string inputType() const = 0;
     const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
 
     const bool hasTarballExtension(std::string_view path) const
@@ -222,22 +221,27 @@ struct CurlInputScheme : InputScheme
         url.query.erase("rev");
         url.query.erase("revCount");
 
-        input.attrs.insert_or_assign("type", inputType());
+        input.attrs.insert_or_assign("type", std::string { schemeName() });
         input.attrs.insert_or_assign("url", url.to_string());
         return input;
     }
 
+    StringSet allowedAttrs() const override
+    {
+        return {
+            "type",
+            "url",
+            "narHash",
+            "name",
+            "unpack",
+            "rev",
+            "revCount",
+            "lastModified",
+        };
+    }
+
     std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
     {
-        auto type = maybeGetStrAttr(attrs, "type");
-        if (type != inputType()) return {};
-
-        // FIXME: some of these only apply to TarballInputScheme.
-        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"};
-        for (auto & [name, value] : attrs)
-            if (!allowedNames.count(name))
-                throw Error("unsupported %s input attribute '%s'", *type, name);
-
         Input input;
         input.attrs = attrs;
 
@@ -258,14 +262,14 @@ struct CurlInputScheme : InputScheme
 
 struct FileInputScheme : CurlInputScheme
 {
-    const std::string inputType() const override { return "file"; }
+    std::string_view schemeName() const override { return "file"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == inputType()
+                ? parsedUrlScheme.application.value() == schemeName()
                 : (!requireTree && !hasTarballExtension(url.path)));
     }
 
@@ -278,7 +282,7 @@ struct FileInputScheme : CurlInputScheme
 
 struct TarballInputScheme : CurlInputScheme
 {
-    const std::string inputType() const override { return "tarball"; }
+    std::string_view schemeName() const override { return "tarball"; }
 
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
@@ -286,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme
 
         return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
             && (parsedUrlScheme.application
-                ? parsedUrlScheme.application.value() == inputType()
+                ? parsedUrlScheme.application.value() == schemeName()
                 : (requireTree || hasTarballExtension(url.path)));
     }
 

src/libmain/common-args.cc

@@ -1,7 +1,9 @@
 #include "common-args.hh"
 #include "args/root.hh"
 #include "globals.hh"
+#include "logging.hh"
 #include "loggers.hh"
+#include "util.hh"
 
 namespace nix {
 

src/libmain/loggers.cc

@@ -1,6 +1,6 @@
 #include "loggers.hh"
+#include "environment-variables.hh"
 #include "progress-bar.hh"
-#include "util.hh"
 
 namespace nix {
 

src/libmain/progress-bar.cc

@@ -1,5 +1,5 @@
 #include "progress-bar.hh"
-#include "util.hh"
+#include "terminal.hh"
 #include "sync.hh"
 #include "store-api.hh"
 #include "names.hh"

src/libmain/shared.cc

@@ -1,10 +1,11 @@
 #include "globals.hh"
+#include "current-process.hh"
 #include "shared.hh"
 #include "store-api.hh"
 #include "gc-store.hh"
-#include "util.hh"
 #include "loggers.hh"
 #include "progress-bar.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <cctype>

src/libmain/shared.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "util.hh"
+#include "processes.hh"
 #include "args.hh"
 #include "args/root.hh"
 #include "common-args.hh"

src/libstore/binary-cache-store.cc

@@ -2,7 +2,7 @@
 #include "binary-cache-store.hh"
 #include "compression.hh"
 #include "derivations.hh"
-#include "fs-accessor.hh"
+#include "source-accessor.hh"
 #include "globals.hh"
 #include "nar-info.hh"
 #include "sync.hh"
@@ -11,6 +11,7 @@
 #include "nar-accessor.hh"
 #include "thread-pool.hh"
 #include "callback.hh"
+#include "signals.hh"
 
 #include <chrono>
 #include <future>
@@ -143,7 +144,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
        write the compressed NAR to disk), into a HashSink (to get the
        NAR hash), and into a NarAccessor (to get the NAR listing). */
     HashSink fileHashSink { htSHA256 };
-    std::shared_ptr<FSAccessor> narAccessor;
+    std::shared_ptr<SourceAccessor> narAccessor;
     HashSink narHashSink { htSHA256 };
     {
     FdSink fileSink(fdTemp.get());
@@ -195,7 +196,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNar(ref<FSAccessor>(narAccessor), "", true)},
+            {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
@@ -206,9 +207,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
        specify the NAR file and member containing the debug info. */
     if (writeDebugInfo) {
 
-        std::string buildIdDir = "/lib/debug/.build-id";
+        CanonPath buildIdDir("lib/debug/.build-id");
 
-        if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
+        if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) {
 
             ThreadPool threadPool(25);
 
@@ -231,17 +232,17 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
             std::regex regex1("^[0-9a-f]{2}$");
             std::regex regex2("^[0-9a-f]{38}\\.debug$");
 
-            for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
+            for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) {
-                auto dir = buildIdDir + "/" + s1;
+                auto dir = buildIdDir + s1;
 
-                if (narAccessor->stat(dir).type != FSAccessor::tDirectory
+                if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory
                     || !std::regex_match(s1, regex1))
                     continue;
 
-                for (auto & s2 : narAccessor->readDirectory(dir)) {
+                for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
-                    auto debugPath = dir + "/" + s2;
+                    auto debugPath = dir + s2;
 
-                    if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
+                    if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
                         || !std::regex_match(s2, regex2))
                         continue;
 
@@ -250,7 +251,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
                     std::string key = "debuginfo/" + buildId;
                     std::string target = "../" + narInfo->url;
 
-                    threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
+                    threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target));
                 }
             }
 
@@ -503,9 +504,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
     upsertFile(filePath, info.toJSON().dump(), "application/json");
 }
 
-ref<FSAccessor> BinaryCacheStore::getFSAccessor()
+ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
 {
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, localNarCache);
 }
 
 void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)

src/libstore/binary-cache-store.hh

@@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
+    const Setting<std::string> compression{this, "xz", "compression",
         "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
 
-    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
+    const Setting<bool> writeNARListing{this, false, "write-nar-listing",
         "Whether to write a JSON file that lists the files in each NAR."};
 
-    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
+    const Setting<bool> writeDebugInfo{this, false, "index-debug-info",
         R"(
           Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
           fetch debug info on demand
         )"};
 
-    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
+    const Setting<Path> secretKeyFile{this, "", "secret-key",
         "Path to the secret key used to sign the binary cache."};
 
-    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
+    const Setting<Path> localNarCache{this, "", "local-nar-cache",
         "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
 
-    const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
+    const Setting<bool> parallelCompression{this, false, "parallel-compression",
         "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
 
-    const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
+    const Setting<int> compressionLevel{this, -1, "compression-level",
         R"(
           The *preset level* to be used when compressing NARs.
           The meaning and accepted values depend on the compression method selected.
@@ -148,7 +148,7 @@ public:
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 
-    ref<FSAccessor> getFSAccessor() override;
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 

src/libstore/build/child.cc

@@ -0,0 +1,37 @@
+#include "child.hh"
+#include "current-process.hh"
+#include "logging.hh"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace nix {
+
+void commonChildInit()
+{
+    logger = makeSimpleLogger();
+
+    const static std::string pathNullDevice = "/dev/null";
+    restoreProcessContext(false);
+
+    /* Put the child in a separate session (and thus a separate
+       process group) so that it has no controlling terminal (meaning
+       that e.g. ssh cannot open /dev/tty) and it doesn't receive
+       terminal signals. */
+    if (setsid() == -1)
+        throw SysError("creating a new session");
+
+    /* Dup stderr to stdout. */
+    if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
+        throw SysError("cannot dup stderr into stdout");
+
+    /* Reroute stdin to /dev/null. */
+    int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
+    if (fdDevNull == -1)
+        throw SysError("cannot open '%1%'", pathNullDevice);
+    if (dup2(fdDevNull, STDIN_FILENO) == -1)
+        throw SysError("cannot dup null device into stdin");
+    close(fdDevNull);
+}
+
+}

src/libstore/build/child.hh

@@ -0,0 +1,11 @@
+#pragma once
+///@file
+
+namespace nix {
+
+/**
+ * Common initialisation performed in child processes.
+ */
+void commonChildInit();
+
+}

src/libstore/build/derivation-goal.cc

@@ -1317,9 +1317,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
                     auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
                     // ensure that logs from a builder using `ssh-ng://` as protocol
                     // are also available to `nix log`.
-                    if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
+                    if (s && !isWrittenToLog && logSink) {
-                        auto f = (*json)["fields"];
+                        const auto type = (*json)["type"];
-                        (*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
+                        const auto fields = (*json)["fields"];
+                        if (type == resBuildLogLine) {
+                            (*logSink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
+                        } else if (type == resSetPhase && ! fields.is_null()) {
+                            const auto phase = fields[0];
+                            if (! phase.is_null()) {
+                                // nixpkgs' stdenv produces lines in the log to signal
+                                // phase changes.
+                                // We want to get the same lines in case of remote builds.
+                                // The format is:
+                                //   @nix { "action": "setPhase", "phase": "$curPhase" }
+                                const auto logLine = nlohmann::json::object({
+                                    {"action", "setPhase"},
+                                    {"phase", phase}
+                                });
+                                (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n");
+                            }
+                        }
                     }
                 }
                 currentHookLine.clear();
@@ -1474,6 +1491,7 @@ void DerivationGoal::done(
     SingleDrvOutputs builtOutputs,
     std::optional<Error> ex)
 {
+    outputLocks.unlock();
     buildResult.status = status;
     if (ex)
         buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));

src/libstore/build/hook-instance.cc

@@ -1,5 +1,7 @@
 #include "globals.hh"
 #include "hook-instance.hh"
+#include "file-system.hh"
+#include "child.hh"
 
 namespace nix {
 

src/libstore/build/hook-instance.hh

@@ -3,6 +3,7 @@
 
 #include "logging.hh"
 #include "serialise.hh"
+#include "processes.hh"
 
 namespace nix {
 

src/libstore/build/local-derivation-goal.cc

@@ -15,7 +15,10 @@
 #include "json-utils.hh"
 #include "cgroup.hh"
 #include "personality.hh"
+#include "current-process.hh"
 #include "namespaces.hh"
+#include "child.hh"
+#include "unix-domain-socket.hh"
 
 #include <regex>
 #include <queue>
@@ -649,8 +652,8 @@ void LocalDerivationGoal::startBuilder()
 #if __linux__
         /* Create a temporary directory in which we set up the chroot
            environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
+           so that the build outputs can be moved efficiently from the
-           inputs in the fake Nix store in the chroot (see below). */
+           chroot to their final location. */
         chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
         deletePath(chrootRootDir);
 
@@ -1563,10 +1566,11 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
             Path source = worker.store.Store::toRealPath(path);
             Path target = chrootRootDir + worker.store.printStorePath(path);
 
-            if (pathExists(target))
+            if (pathExists(target)) {
                 // There is a similar debug message in doBind, so only run it in this block to not have double messages.
                 debug("bind-mounting %s -> %s", target, source);
                 throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
+            }
 
             /* Bind-mount the path into the sandbox. This requires
                entering its mount namespace, which is not possible
@@ -1619,6 +1623,8 @@ void setupSeccomp()
         seccomp_release(ctx);
     });
 
+    constexpr std::string_view nativeSystem = SYSTEM;
+
     if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");

src/libstore/build/local-derivation-goal.hh

@@ -3,6 +3,7 @@
 
 #include "derivation-goal.hh"
 #include "local-store.hh"
+#include "processes.hh"
 
 namespace nix {
 

src/libstore/build/worker.cc

@@ -4,6 +4,7 @@
 #include "drv-output-substitution-goal.hh"
 #include "local-derivation-goal.hh"
 #include "hook-instance.hh"
+#include "signals.hh"
 
 #include <poll.h>
 

src/libstore/common-protocol.cc

@@ -1,5 +1,4 @@
 #include "serialise.hh"
-#include "util.hh"
 #include "path-with-outputs.hh"
 #include "store-api.hh"
 #include "build-result.hh"

src/libstore/crypto.cc

@@ -1,4 +1,5 @@
 #include "crypto.hh"
+#include "file-system.hh"
 #include "util.hh"
 #include "globals.hh"
 

src/libstore/daemon.cc

@@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                        eagerly consume the entire stream it's given, past the
                        length of the Nar. */
                     TeeSource savedNARSource(from, saved);
-                    ParseSink sink; /* null sink; just parse the NAR */
+                    NullParseSink sink; /* just parse the NAR */
                     parseDump(sink, savedNARSource);
                 } else {
                     /* Incrementally parse the NAR file, stripping the
                        metadata, and streaming the sole file we expect into
                        `saved`. */
-                    RetrieveRegularNARSink savedRegular { saved };
+                    RegularFileSink savedRegular { saved };
                     parseDump(savedRegular, from);
                     if (!savedRegular.regular) throw Error("regular file expected");
                 }
@@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 source = std::make_unique<TunnelSource>(from, to);
             else {
                 TeeSource tee { from, saved };
-                ParseSink ether;
+                NullParseSink ether;
                 parseDump(ether, tee);
                 source = std::make_unique<StringSource>(saved.s);
             }