Merge remote-tracking branch 'origin/master' into nix-profile-stable-names

.clang-format

@@ -0,0 +1,30 @@
+BasedOnStyle: LLVM
+IndentWidth: 4
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterStruct: true
+  AfterClass: true
+  AfterFunction: true
+  AfterUnion: true
+  SplitEmptyRecord: false
+PointerAlignment: Middle
+FixNamespaceComments: false
+SortIncludes: Never
+#IndentPPDirectives: BeforeHash
+SpaceAfterCStyleCast: true
+SpaceAfterTemplateKeyword: false
+AccessModifierOffset: -4
+AlignAfterOpenBracket: AlwaysBreak
+AlignEscapedNewlines: DontAlign
+ColumnLimit: 120
+BreakStringLiterals: false
+BitFieldColonSpacing: None
+AllowShortFunctionsOnASingleLine: Empty
+AlwaysBreakTemplateDeclarations: Yes
+BinPackParameters: false
+BreakConstructorInitializers: BeforeComma
+EmptyLineAfterAccessModifier: Leave # change to always/never later?
+EmptyLineBeforeAccessModifier: Leave
+#PackConstructorInitializers: BinPack
+BreakBeforeBinaryOperators: NonAssignment
+AlwaysBreakBeforeMultilineStrings: true

.gitignore

@@ -141,6 +141,7 @@ compile_commands.json
 nix-rust/target
 
 result
+result-*
 
 # IDE
 .vscode/

Makefile

@@ -1,8 +1,12 @@
+# External build directory support
+
 include mk/build-dir.mk
 
 -include $(buildprefix)Makefile.config
 clean-files += $(buildprefix)Makefile.config
 
+# List makefiles
+
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
   mk/precompiled-headers.mk \
@@ -43,6 +47,8 @@ makefiles += \
   tests/functional/plugins/local.mk
 endif
 
+# Miscellaneous global Flags
+
 OPTIMIZE = 1
 
 ifeq ($(OPTIMIZE), 1)
@@ -52,9 +58,29 @@ else
   GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
 endif
 
+include mk/platform.mk
+
+ifdef HOST_WINDOWS
+  # Windows DLLs are stricter about symbol visibility than Unix shared
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
+  # This is a temporary sledgehammer to export everything like on Unix,
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  GLOBAL_LDFLAGS += -Wl,--export-all-symbols
+endif
+
+GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
+
+# Include the main lib, causing rules to be defined
+
 include mk/lib.mk
 
-# Must be included after `mk/lib.mk` so isn't the default target.
+# Fallback stub rules for better UX when things are disabled
+#
+# These must be defined after `mk/lib.mk`. Otherwise the first rule
+# incorrectly becomes the default target.
+
 ifneq ($(ENABLE_UNIT_TESTS), yes)
 .PHONY: check
 check:
@@ -69,8 +95,11 @@ installcheck:
 	@exit 1
 endif
 
-# Must be included after `mk/lib.mk` so rules refer to variables defined
+# Documentation or else fallback stub rules.
-# by the library. Rules are not "lazy" like variables, unfortunately.
+#
+# The documentation makefiles be included after `mk/lib.mk` so rules
+# refer to variables defined by `mk/lib.mk`. Rules are not "lazy" like
+# variables, unfortunately.
 
 ifeq ($(ENABLE_DOC_GEN), yes)
 $(eval $(call include-sub-makefile, doc/manual/local.mk))
@@ -89,5 +118,3 @@ internal-api-html:
 	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
 	@exit 1
 endif
-
-GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src

configure.ac

@@ -160,7 +160,7 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
 AC_SUBST(ENABLE_DOC_GEN)
 
 AS_IF(
-  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_GENERATED_DOCS" == "yes"],
+  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"],
   [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])])
 
 # Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
@@ -251,17 +251,25 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
-# Look for editline, a required dependency.
+# Look for editline or readline, a required dependency.
 # The the libeditline.pc file was added only in libeditline >= 1.15.2,
 # see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607,
-# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for
+# Older versions are no longer supported.
-# editline.h when the pkg-config approach fails.
+AC_ARG_WITH(
-PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [
+  [readline-flavor],
-  AC_CHECK_HEADERS([editline.h], [true],
+  AS_HELP_STRING([--with-readline-flavor],[Which library to use for nice line editting with the Nix language REPL" [default=editline]]),
-    [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])])
+  [readline_flavor=$withval],
-  AC_SEARCH_LIBS([readline read_history], [editline], [],
+  [readline_flavor=editline])
-    [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
+AS_CASE(["$readline_flavor"],
-])
+  [editline], [
+    readline_flavor_pc=libeditline
+  ],
+  [readline], [
+    readline_flavor_pc=readline
+    AC_DEFINE([USE_READLINE], [1], [Use readline instead of editline])
+  ],
+  [AC_MSG_ERROR([bad value "$readline_flavor" for --with-readline-flavor, must be one of: editline, readline])])
+PKG_CHECK_MODULES([EDITLINE], [$readline_flavor_pc], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
 
 # Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
@@ -308,7 +316,12 @@ AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 # Optional dependencies for better normalizing file system data
 AC_CHECK_HEADERS([sys/xattr.h])
-AC_CHECK_FUNCS([llistxattr lremovexattr])
+AS_IF([test "$ac_cv_header_sys_xattr_h" = "yes"],[
+  AC_CHECK_FUNCS([llistxattr lremovexattr])
+  AS_IF([test "$ac_cv_func_llistxattr" = "yes" && test "$ac_cv_func_lremovexattr" = "yes"],[
+    AC_DEFINE([HAVE_ACL_SUPPORT], [1], [Define if we can manipulate file system Access Control Lists])
+  ])
+])
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
@@ -369,7 +382,20 @@ PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 
 
 # Look for lowdown library.
-PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
+AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]),
+  enable_markdown=$enableval, enable_markdown=auto)
+AS_CASE(["$enable_markdown"],
+  [yes | auto], [
+    PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [
+      CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"
+      have_lowdown=1
+      AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.])
+    ], [
+      AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])])
+    ])
+  ],
+  [no], [have_lowdown=],
+  [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])])
 
 
 # Look for libgit2.

doc/manual/redirects.js

@@ -21,6 +21,7 @@ const redirects = {
     "chap-distributed-builds": "advanced-topics/distributed-builds.html",
     "chap-post-build-hook": "advanced-topics/post-build-hook.html",
     "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
+    "chap-writing-nix-expressions": "language/index.html",
     "part-command-ref": "command-ref/command-ref.html",
     "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
     "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",

doc/manual/rl-next/drv-string-parse-hang.md

@@ -0,0 +1,6 @@
+---
+synopsis: Fix handling of truncated `.drv` files.
+prs: 9673
+---
+
+Previously a `.drv` that was truncated in the middle of a string would case nix to enter an infinite loop, eventually exhausting all memory and crashing.

doc/manual/rl-next/env-size-reduction.md

@@ -0,0 +1,7 @@
+---
+synopsis: Reduce eval memory usage and wall time
+prs: 9658
+---
+
+Reduce the size of the `Env` struct used in the evaluator by a pointer, or 8 bytes on most modern machines.
+This reduces memory usage during eval by around 2% and wall time by around 3%.

doc/manual/rl-next/ifd-eval-store.md

@@ -0,0 +1,8 @@
+---
+synopsis: import-from-derivation builds the derivation in the build store
+prs: 9661
+---
+
+When using `--eval-store`, `import`ing from a derivation will now result in the derivation being built on the build store, i.e. the store specified in the `store` Nix option.
+
+Because the resulting Nix expression must be copied back to the eval store in order to be imported, this requires the eval store to trust the build store's signatures.

doc/manual/rl-next/stack-overflow-segfaults.md

@@ -0,0 +1,32 @@
+---
+synopsis: Some stack overflow segfaults are fixed
+issues: 9616
+prs: 9617
+---
+
+The number of nested function calls has been restricted, to detect and report
+infinite function call recursions. The default maximum call depth is 10,000 and
+can be set with [the `max-call-depth`
+option](@docroot@/command-ref/conf-file.md#conf-max-call-depth).
+
+This fixes segfaults or the following unhelpful error message in many cases:
+
+    error: stack overflow (possible infinite recursion)
+
+Before:
+
+```
+$ nix-instantiate --eval --expr '(x: x x) (x: x x)'
+Segmentation fault: 11
+```
+
+After:
+
+```
+$ nix-instantiate --eval --expr '(x: x x) (x: x x)'
+error: stack overflow
+
+       at «string»:1:14:
+            1| (x: x x) (x: x x)
+             |              ^
+```

doc/manual/rl-next/with-error-reporting.md

@@ -0,0 +1,31 @@
+---
+synopsis: Better error reporting for `with` expressions
+prs: 9658
+---
+
+`with` expressions using non-attrset values to resolve variables are now reported with proper positions.
+
+Previously an incorrect `with` expression would report no position at all, making it hard to determine where the error originated:
+
+```
+nix-repl> with 1; a  
+error:
+       … <borked>
+
+         at «none»:0: (source not available)
+
+       error: value is an integer while a set was expected
+```
+
+Now position information is preserved and reported as with most other errors:
+
+```
+nix-repl> with 1; a
+error:
+       … while evaluating the first subexpression of a with expression
+         at «string»:1:1:
+            1| with 1; a
+             | ^
+
+       error: value is an integer while a set was expected
+```

doc/manual/src/command-ref/nix-instantiate.md

@@ -35,13 +35,50 @@ standard input.
 
   - `--parse`\
     Just parse the input files, and print their abstract syntax trees on
-    standard output in ATerm format.
+    standard output as a Nix expression.
 
   - `--eval`\
     Just parse and evaluate the input files, and print the resulting
     values on standard output. No instantiation of store derivations
     takes place.
 
+    > **Warning**
+    >
+    > This option produces ambiguous output which is not suitable for machine
+    > consumption. For example, these two Nix expressions print the same result
+    > despite having different types:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr '{ a = {}; }'
+    > { a = <CODE>; }
+    > $ nix-instantiate --eval --expr '{ a = <CODE>; }'
+    > { a = <CODE>; }
+    > ```
+    >
+    > For human-readable output, `nix eval` (experimental) is more informative:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr 'a: a'
+    > <LAMBDA>
+    > $ nix eval --expr 'a: a'
+    > «lambda @ «string»:1:1»
+    > ```
+    >
+    > For machine-readable output, the `--xml` option produces unambiguous
+    > output:
+    >
+    > ```console
+    > $ nix-instantiate --eval --xml --expr '{ foo = <CODE>; }'
+    > <?xml version='1.0' encoding='utf-8'?>
+    > <expr>
+    >   <attrs>
+    >     <attr column="3" line="1" name="foo">
+    >       <unevaluated />
+    >     </attr>
+    >   </attrs>
+    > </expr>
+    > ```
+
   - `--find-file`\
     Look up the given files in Nix’s search path (as specified by the
     `NIX_PATH` environment variable). If found, print the corresponding
@@ -61,11 +98,11 @@ standard input.
 
   - `--json`\
     When used with `--eval`, print the resulting value as an JSON
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
 
   - `--xml`\
     When used with `--eval`, print the resulting value as an XML
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
     The schema is the same as that used by the [`toXML`
     built-in](../language/builtins.md).
 
@@ -133,28 +170,29 @@ $ nix-instantiate --eval --xml --expr '1 + 2'
 The difference between non-strict and strict evaluation:
 
 ```console
-$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }'
+$ nix-instantiate --eval --xml --expr '{ x = {}; }'
-...
+<?xml version='1.0' encoding='utf-8'?>
-  <attr name="x">
+<expr>
-    <string value="foo" />
+  <attrs>
-  </attr>
+    <attr column="3" line="1" name="x">
-  <attr name="y">
       <unevaluated />
     </attr>
-...
+  </attrs>
+</expr>
 ```
 
 Note that `y` is left unevaluated (the XML representation doesn’t
 attempt to show non-normal forms).
 
 ```console
-$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }'
+$ nix-instantiate --eval --xml --strict --expr '{ x = {}; }'
-...
+<?xml version='1.0' encoding='utf-8'?>
-  <attr name="x">
+<expr>
-    <string value="foo" />
+  <attrs>
+    <attr column="3" line="1" name="x">
+      <attrs>
+      </attrs>
     </attr>
-  <attr name="y">
+  </attrs>
-    <string value="foo" />
+</expr>
-  </attr>
-...
 ```

doc/manual/src/contributing/hacking.md

@@ -31,7 +31,7 @@ This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` im
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix develop .#native-clang11StdenvPackages
+$ nix develop .#native-clangStdenvPackages
 ```
 
 > **Note**
@@ -96,7 +96,7 @@ $ nix-shell
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages
+$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages
 ```
 
 > **Note**

doc/manual/src/installation/installing-binary.md

@@ -1,26 +1,60 @@
 # Installing a Binary Distribution
 
-The easiest way to install Nix is to run the following command:
+To install the latest version Nix, run the following command:
 
 ```console
 $ curl -L https://nixos.org/nix/install | sh
 ```
 
-This will run the installer interactively (causing it to explain what
+This performs the default type of installation for your platform:
-it is doing more explicitly), and perform the default "type" of install
-for your platform:
-- single-user on Linux
-- multi-user on macOS
 
-  > **Notes on read-only filesystem root in macOS 10.15 Catalina +**
+- [Multi-user](#multi-user-installation):
-  >
+  - Linux with systemd and without SELinux
-  > - It took some time to support this cleanly. You may see posts,
+  - macOS
-  >   examples, and tutorials using obsolete workarounds.
+- [Single-user](#single-user-installation):
-  > - Supporting it cleanly made macOS installs too complex to qualify
+  - Linux without systemd
-  >   as single-user, so this type is no longer supported on macOS.
+  - Linux with SELinux
 
-We recommend the multi-user install if it supports your platform and
+We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`.
-you can authenticate with `sudo`.
+
+The installer can configured with various command line arguments and environment variables.
+To show available command line flags:
+
+```console
+$ curl -L https://nixos.org/nix/install | sh -s -- --help
+```
+
+To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball).
+
+# Installing a pinned Nix version from a URL
+
+Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/).
+The directory for each version contains the corresponding SHA-256 hash.
+
+All installation scripts are invoked the same way:
+
+```console
+$ export VERSION=2.19.2 
+$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh
+```
+
+# Multi User Installation
+
+The multi-user Nix installation creates system users and a system service for the Nix daemon.
+
+Supported systems:
+
+- Linux running systemd, with SELinux disabled
+- macOS
+
+To explicitly instruct the installer to perform a multi-user installation on your system:
+
+```console
+$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
+```
+
+You can run this under your usual user account or `root`.
+The script will invoke `sudo` as needed.
 
 # Single User Installation
 
@@ -30,60 +64,48 @@ To explicitly select a single-user installation on your system:
 $ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon
 ```
 
-This will perform a single-user installation of Nix, meaning that `/nix`
+In a single-user installation, `/nix` is owned by the invoking user.
-is owned by the invoking user. You can run this under your usual user
+The script will invoke `sudo` to create `/nix` if it doesn’t already exist.
-account or root. The script will invoke `sudo` to create `/nix`
+If you don’t have `sudo`, manually create `/nix` as `root`:
-if it doesn’t already exist. If you don’t have `sudo`, you should
-manually create `/nix` first as root, e.g.:
 
 ```console
-$ mkdir /nix
+$ su root
-$ chown alice /nix
+# mkdir /nix
+# chown alice /nix
 ```
 
-The install script will modify the first writable file from amongst
+# Installing from a binary tarball
-`.bash_profile`, `.bash_login` and `.profile` to source
-`~/.nix-profile/etc/profile.d/nix.sh`. You can set the
-`NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing
-the install script to disable this behaviour.
 
-# Multi User Installation
+You can also download a binary tarball that contains Nix and all its dependencies:
+- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms)
+- Download and unpack the tarball
+- Run the installer
 
-The multi-user Nix installation creates system users, and a system
+> **Example**
-service for the Nix daemon.
-
-**Supported Systems**
-- Linux running systemd, with SELinux disabled
-- macOS
-
-You can instruct the installer to perform a multi-user installation on
-your system:
-
-```console
-$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
-```
-
-The multi-user installation of Nix will create build users between the
-user IDs 30001 and 30032, and a group with the group ID 30000. You
-can run this under your usual user account or root. The script
-will invoke `sudo` as needed.
-
-> **Note**
 >
-> If you need Nix to use a different group ID or user ID set, you will
+> ```console
-> have to download the tarball manually and [edit the install
+> $ pushd $(mktemp -d)
-> script](#installing-from-a-binary-tarball).
+> $ export VERSION=2.19.2
+> $ export SYSTEM=x86_64-linux
+> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz
+> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz
+> $ cd nix-$VERSION-$SYSTEM
+> $ ./install
+> $ popd
+> ```
 
-The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
+The installer can be customised with the environment variables declared in the file named `install-multi-user`.
-The installer will first back up these files with a `.backup-before-nix`
+
-extension. The installer will also create `/etc/profile.d/nix.sh`.
+## Native packages for Linux distributions
+
+The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/).
 
 # macOS Installation
 
+<!-- anchors to catch existing links -->
 []{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes}
-<!-- Note: anchors above to catch permalinks to old explanations -->
 
-We believe we have ironed out how to cleanly support the read-only root
+We believe we have ironed out how to cleanly support the read-only root file system
 on modern macOS. New installs will do this automatically.
 
 This section previously detailed the situation, options, and trade-offs,
@@ -126,33 +148,3 @@ this to run the installer, but it may help if you run into trouble:
   boot process to avoid problems loading or restoring any programs that
   need access to your Nix store
 
-# Installing a pinned Nix version from a URL
-
-Version-specific installation URLs for all Nix versions
-since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/).
-The corresponding SHA-256 hash can be found in the directory for the given version.
-
-These install scripts can be used the same as usual:
-
-```console
-$ curl -L https://releases.nixos.org/nix/nix-<version>/install | sh
-```
-
-# Installing from a binary tarball
-
-You can also download a binary tarball that contains Nix and all its
-dependencies. (This is what the install script at
-<https://nixos.org/nix/install> does automatically.) You should unpack
-it somewhere (e.g. in `/tmp`), and then run the script named `install`
-inside the binary tarball:
-
-```console
-$ cd /tmp
-$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-$ cd nix-1.8-x86_64-darwin
-$ ./install
-```
-
-If you need to edit the multi-user installation script to use different
-group ID or a different user ID range, modify the variables set in the
-file named `install-multi-user`.

doc/manual/src/installation/upgrading.md

@@ -1,5 +1,40 @@
 # Upgrading Nix
 
+> **Note**
+>
+> These upgrade instructions apply for regular Linux distributions where Nix was installed following the [installation instructions in this manual](./index.md).
+
+First, find the name of the current [channel](@docroot@/command-ref/nix-channel.md) through which Nix is distributed:
+
+```console
+$ nix-channel --list
+```
+
+By default this should return an entry for Nixpkgs:
+
+```console
+nixpkgs https://nixos.org/channels/nixpkgs-23.05
+```
+
+Check which Nix version will be installed:
+
+```console
+$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-23.11 --run "nix --version"
+nix (Nix) 2.18.1
+```
+
+> **Warning**
+>
+> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with `nix-build` or `nix-store --realise`, may change the database schema!
+> Reverting to an older version of Nix may therefore require purging the store database before it can be used.
+
+Update the channel entry:
+
+```console
+$ nix-channel --remove nixpkgs
+$ nix-channel --add https://nixos.org/channels/nixpkgs-23.11 nixpkgs
+```
+
 Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c
 'nix-channel --update &&
 nix-env --install --attr nixpkgs.nix &&

doc/manual/src/language/derivations.md

@@ -274,7 +274,7 @@ The [`builder`](#attr-builder) is executed as follows:
     directory (typically, `/nix/store`).
 
   - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs`
-    is set to `true` for the dervation. A detailed explanation of this
+    is set to `true` for the derivation. A detailed explanation of this
     behavior can be found in the
     [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs).
 

doc/manual/src/quick-start.md

@@ -10,7 +10,6 @@ For more in-depth information you are kindly referred to subsequent chapters.
    ```
 
    The install script will use `sudo`, so make sure you have sufficient rights.
-   On Linux, `--daemon` can be omitted for a single-user install.
 
    For other installation methods, see the detailed [installation instructions](installation/index.md).
 

flake.lock

@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1701355166,
+        "lastModified": 1704018918,
-        "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=",
+        "narHash": "sha256-erjg/HrpC9liEfm7oLqb8GXCqsxaFwIIPqCsknW5aFY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c",
+        "rev": "2c9c58e98243930f8cb70387934daa4bc8b00373",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "staging-23.05",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,17 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  # TODO Go back to nixos-23.05-small once
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  # https://github.com/NixOS/nixpkgs/pull/271202 is merged.
-  #
-  # Also, do not grab arbitrary further staging commits. This PR was
-  # carefully made to be based on release-23.05 and just contain
-  # rebuild-causing changes to packages that Nix actually uses.
-  #
-  # Once this is updated to something containing
-  # https://github.com/NixOS/nixpkgs/pull/271423, don't forget
-  # to remove the `nix.checkAllErrors = false;` line in the tests.
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
@@ -62,7 +52,6 @@
 
       stdenvs = [
         "ccacheStdenv"
-        "clang11Stdenv"
         "clangStdenv"
         "gccStdenv"
         "libcxxStdenv"
@@ -231,14 +220,25 @@
         buildCross = forAllCrossSystems (crossSystem:
           lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
 
-        buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
+        buildNoGc = forAllSystems (system:
+          self.packages.${system}.nix.override { enableGC = false; }
+        );
 
         buildNoTests = forAllSystems (system:
-          self.packages.${system}.nix.overrideAttrs (a: {
+          self.packages.${system}.nix.override {
-            doCheck =
+            doCheck = false;
-              assert ! a?dontCheck;
+            doInstallCheck = false;
-              false;
+            installUnitTests = false;
-          })
+          }
+        );
+
+        # Toggles some settings for better coverage. Windows needs these
+        # library combinations, and Debian build Nix with GNU readline too.
+        buildReadlineNoMarkdown = forAllSystems (system:
+          self.packages.${system}.nix.override {
+            enableMarkdown = false;
+            readlineFlavor = "readline";
+          }
         );
 
         # Perl bindings for various platforms.

mk/lib.mk

@@ -12,24 +12,7 @@ man-pages :=
 install-tests :=
 install-tests-groups :=
 
-ifdef HOST_OS
+include mk/platform.mk
-  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
-  ifeq ($(HOST_KERNEL), cygwin)
-    HOST_CYGWIN = 1
-  endif
-  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
-    HOST_DARWIN = 1
-  endif
-  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
-    HOST_FREEBSD = 1
-  endif
-  ifeq ($(HOST_KERNEL), linux)
-    HOST_LINUX = 1
-  endif
-  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
-    HOST_SOLARIS = 1
-  endif
-endif
 
 # Hack to define a literal space.
 space :=

mk/libraries.mk

@@ -3,13 +3,19 @@ libs-list :=
 ifdef HOST_DARWIN
   SO_EXT = dylib
 else
-  ifdef HOST_CYGWIN
+  ifdef HOST_WINDOWS
     SO_EXT = dll
   else
     SO_EXT = so
   endif
 endif
 
+ifdef HOST_UNIX
+  THREAD_LDFLAGS = -pthread
+else
+  THREAD_LDFLAGS =
+endif
+
 # Build a library with symbolic name $(1).  The library is defined by
 # various variables prefixed by ‘$(1)_’:
 #
@@ -59,7 +65,7 @@ define build-library
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH))
 
-  ifdef HOST_CYGWIN
+  ifdef HOST_WINDOWS
     $(1)_INSTALL_DIR ?= $$(bindir)
   else
     $(1)_INSTALL_DIR ?= $$(libdir)
@@ -79,7 +85,7 @@ define build-library
       endif
     else
       ifndef HOST_DARWIN
-        ifndef HOST_CYGWIN
+        ifndef HOST_WINDOWS
           $(1)_LDFLAGS += -Wl,-z,defs
         endif
       endif

mk/platform.mk

@@ -0,0 +1,32 @@
+ifdef HOST_OS
+  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
+  ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),)
+    HOST_MINGW = 1
+    HOST_WINDOWS = 1
+  endif
+  ifeq ($(HOST_KERNEL), cygwin)
+    HOST_CYGWIN = 1
+    HOST_WINDOWS = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
+    HOST_DARWIN = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
+    HOST_FREEBSD = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),)
+    HOST_NETBSD = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(HOST_KERNEL), linux)
+    HOST_LINUX = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
+    HOST_SOLARIS = 1
+    HOST_UNIX = 1
+  endif
+endif

mk/programs.mk

@@ -1,5 +1,11 @@
 programs-list :=
 
+ifdef HOST_WINDOWS
+  EXE_EXT = .exe
+else
+  EXE_EXT =
+endif
+
 # Build a program with symbolic name $(1).  The program is defined by
 # various variables prefixed by ‘$(1)_’:
 #
@@ -31,7 +37,7 @@ define build-program
   _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src)))
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH)))
-  $(1)_PATH := $$(_d)/$$($(1)_NAME)
+  $(1)_PATH := $$(_d)/$$($(1)_NAME)$(EXE_EXT)
 
   $$(eval $$(call create-dir, $$(_d)))
 
@@ -42,7 +48,7 @@ define build-program
 
   ifdef $(1)_INSTALL_DIR
 
-    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)
+    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)$(EXE_EXT)
 
     $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 

package.nix

@@ -13,6 +13,7 @@
 , changelog-d
 , curl
 , editline
+, readline
 , fileset
 , flex
 , git
@@ -68,6 +69,25 @@
 # Whether to build the regular manual
 , enableManual ? __forDefaults.canRunInstalled
 
+# Whether to use garbage collection for the Nix language evaluator.
+#
+# If it is disabled, we just leak memory, but this is not as bad as it
+# sounds so long as evaluation just takes places within short-lived
+# processes. (When the process exits, the memory is reclaimed; it is
+# only leaked *within* the process.)
+, enableGC ? true
+
+# Whether to enable Markdown rendering in the Nix binary.
+, enableMarkdown ? !stdenv.hostPlatform.isWindows
+
+# Which interactive line editor library to use for Nix's repl.
+#
+# Currently supported choices are:
+#
+# - editline (default)
+# - readline
+, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+
 # Whether to compile `rl-next.md`, the release notes for the next
 # not-yet-released version of Nix in the manul, from the individual
 # change log entries in the directory.
@@ -80,7 +100,7 @@
 # Whether to install unit tests. This is useful when cross compiling
 # since we cannot run them natively during the build, but can do so
 # later.
-, installUnitTests ? __forDefaults.canRunInstalled
+, installUnitTests ? doBuild && !__forDefaults.canExecuteHost
 
 # For running the functional tests against a pre-built Nix. Probably
 # want to use in conjunction with `doBuild = false;`.
@@ -93,7 +113,8 @@
 # Not a real argument, just the only way to approximate let-binding some
 # stuff for argument defaults.
 , __forDefaults ? {
-    canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+    canExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+    canRunInstalled = doBuild && __forDefaults.canExecuteHost;
   }
 }:
 
@@ -164,6 +185,10 @@ in {
           ./doc/manual
         ] ++ lib.optionals enableInternalAPIDocs [
           ./doc/internal-api
+          # Source might not be compiled, but still must be available
+          # for Doxygen to gather comments.
+          ./src
+          ./tests/unit
         ] ++ lib.optionals buildUnitTests [
           ./tests/unit
         ] ++ lib.optionals doInstallCheck [
@@ -212,8 +237,12 @@ in {
     sqlite
     xz
   ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
-    editline
+    ({ inherit readline editline; }.${readlineFlavor})
+  ] ++ lib.optionals enableMarkdown [
     lowdown
+  ] ++ lib.optionals buildUnitTests [
+    gtest
+    rapidcheck
   ] ++ lib.optional stdenv.isLinux libseccomp
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
@@ -225,18 +254,12 @@ in {
   ;
 
   propagatedBuildInputs = [
-    boehmgc
     nlohmann_json
-  ];
+  ] ++ lib.optional enableGC boehmgc;
 
   dontBuild = !attrs.doBuild;
   doCheck = attrs.doCheck;
 
-  checkInputs = [
-    gtest
-    rapidcheck
-  ];
-
   nativeCheckInputs = [
     git
     mercurial
@@ -250,7 +273,7 @@ in {
       # Copy libboost_context so we don't get all of Boost in our closure.
       # https://github.com/NixOS/nixpkgs/issues/45462
       mkdir -p $out/lib
-      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
+      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
       rm -f $out/lib/*.a
     '' + lib.optionalString stdenv.hostPlatform.isLinux ''
       chmod u+w $out/lib/*.so.*
@@ -271,7 +294,10 @@ in {
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
     (lib.enableFeature enableManual "doc-gen")
+    (lib.enableFeature enableGC "gc")
+    (lib.enableFeature enableMarkdown "markdown")
     (lib.enableFeature installUnitTests "install-unit-tests")
+    (lib.withFeatureAs true "readline-flavor" readlineFlavor)
   ] ++ lib.optionals (!forDevShell) [
     "--sysconfdir=/etc"
   ] ++ lib.optionals installUnitTests [

perl/lib/Nix/Store.xs

@@ -12,7 +12,6 @@
 #include "realisation.hh"
 #include "globals.hh"
 #include "store-api.hh"
-#include "crypto.hh"
 #include "posix-source-accessor.hh"
 
 #include <sodium.h>

src/libcmd/built-path.cc

@@ -12,9 +12,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \

src/libcmd/editor-for.cc

@@ -1,5 +1,6 @@
 #include "editor-for.hh"
 #include "environment-variables.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libcmd/editor-for.hh

@@ -2,7 +2,7 @@
 ///@file
 
 #include "types.hh"
-#include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libcmd/installable-flake.cc

@@ -52,7 +52,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
     auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
     assert(aOutputs);
 
-    state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
+    state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
 
     return aOutputs->value;
 }

src/libcmd/installable-value.cc

@@ -1,5 +1,6 @@
 #include "installable-value.hh"
 #include "eval-cache.hh"
+#include "fetch-to-store.hh"
 
 namespace nix {
 
@@ -44,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(*state->store);
+        auto storePath = fetchToStore(*state->store, v.path());
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libcmd/installables.cc

@@ -715,7 +715,7 @@ BuiltPaths Installable::toBuiltPaths(
     }
 }
 
-StorePathSet Installable::toStorePaths(
+StorePathSet Installable::toStorePathSet(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -729,13 +729,27 @@ StorePathSet Installable::toStorePaths(
     return outPaths;
 }
 
+StorePaths Installable::toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode, OperateOn operateOn,
+    const Installables & installables)
+{
+    StorePaths outPaths;
+    for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
+        auto thisOutPaths = path.outPaths();
+        outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end());
+    }
+    return outPaths;
+}
+
 StorePath Installable::toStorePath(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
     ref<Installable> installable)
 {
-    auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
+    auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable});
 
     if (paths.size() != 1)
         throw Error("argument '%s' should evaluate to one store path", installable->what());

src/libcmd/installables.hh

@@ -165,7 +165,14 @@ struct Installable
         const Installables & installables,
         BuildMode bMode = bmNormal);
 
-    static std::set<StorePath> toStorePaths(
+    static std::set<StorePath> toStorePathSet(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const Installables & installables);
+
+    static std::vector<StorePath> toStorePaths(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,

src/libcmd/local.mk

@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
+libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 

src/libcmd/markdown.cc

@@ -4,12 +4,15 @@
 #include "terminal.hh"
 
 #include <sys/queue.h>
+#if HAVE_LOWDOWN
 #include <lowdown.h>
+#endif
 
 namespace nix {
 
 std::string renderMarkdownToTerminal(std::string_view markdown)
 {
+#if HAVE_LOWDOWN
     int windowWidth = getWindowSize().second;
 
     struct lowdown_opts opts {
@@ -48,6 +51,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("allocation error while rendering Markdown");
 
     return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
+#else
+    return std::string(markdown);
+#endif
 }
 
 }

src/libcmd/repl.cc

@@ -5,7 +5,7 @@
 
 #include <setjmp.h>
 
-#ifdef READLINE
+#ifdef USE_READLINE
 #include <readline/history.h>
 #include <readline/readline.h>
 #else
@@ -93,9 +93,17 @@ struct NixRepl
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
-    typedef std::set<Value *> ValuesSeen;
+    void printValue(std::ostream & str,
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth);
+                              Value & v,
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen);
+                              unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
+    {
+        ::nix::printValue(*state, str, v, PrintOptions {
+            .ansiColors = true,
+            .force = true,
+            .derivationPaths = true,
+            .maxDepth = maxDepth
+        });
+    }
 };
 
 std::string removeWhitespace(std::string s)
@@ -112,7 +120,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
-    , staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
+    , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
     , historyFile(getDataDir() + "/nix/repl-history")
 {
 }
@@ -221,7 +229,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     // prefer direct pos, but if noPos then try the expr.
     auto pos = dt.pos
         ? dt.pos
-        : static_cast<std::shared_ptr<AbstractPos>>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]);
+        : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
 
     if (pos) {
         out << pos;
@@ -249,14 +257,14 @@ void NixRepl::mainLoop()
     } catch (SysError & e) {
         logWarning(e.info());
     }
-#ifndef READLINE
+#ifndef USE_READLINE
     el_hist_size = 1000;
 #endif
     read_history(historyFile.c_str());
     auto oldRepl = curRepl;
     curRepl = this;
     Finally restoreRepl([&] { curRepl = oldRepl; });
-#ifndef READLINE
+#ifndef USE_READLINE
     rl_set_complete_func(completionCallback);
     rl_set_list_possib_func(listPossibleCallback);
 #endif
@@ -708,7 +716,8 @@ bool NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
-        printValue(std::cout, v, 1000000000) << std::endl;
+        printValue(std::cout, v);
+        std::cout << std::endl;
     }
 
     else if (command == ":q" || command == ":quit") {
@@ -770,7 +779,8 @@ bool NixRepl::processLine(std::string line)
         } else {
             Value v;
             evalString(line, v);
-            printValue(std::cout, v, 1) << std::endl;
+            printValue(std::cout, v, 1);
+            std::cout << std::endl;
         }
     }
 
@@ -888,145 +898,7 @@ void NixRepl::evalString(std::string s, Value & v)
 {
     Expr * e = parseString(s);
     e->eval(*state, *env, v);
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
-}
-
-
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth)
-{
-    ValuesSeen seen;
-    return printValue(str, v, maxDepth, seen);
-}
-
-
-
-
-// FIXME: lot of cut&paste from Nix's eval.cc.
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen)
-{
-    str.flush();
-    checkInterrupt();
-
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
-
-    switch (v.type()) {
-
-    case nInt:
-        str << ANSI_CYAN << v.integer << ANSI_NORMAL;
-        break;
-
-    case nBool:
-        str << ANSI_CYAN;
-        printLiteralBool(str, v.boolean);
-        str << ANSI_NORMAL;
-        break;
-
-    case nString:
-        str << ANSI_WARNING;
-        printLiteralString(str, v.string_view());
-        str << ANSI_NORMAL;
-        break;
-
-    case nPath:
-        str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping?
-        break;
-
-    case nNull:
-        str << ANSI_CYAN "null" ANSI_NORMAL;
-        break;
-
-    case nAttrs: {
-        seen.insert(&v);
-
-        bool isDrv = state->isDerivation(v);
-
-        if (isDrv) {
-            str << "«derivation ";
-            Bindings::iterator i = v.attrs->find(state->sDrvPath);
-            NixStringContext context;
-            if (i != v.attrs->end())
-                str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
-            else
-                str << "???";
-            str << "»";
-        }
-
-        else if (maxDepth > 0) {
-            str << "{ ";
-
-            typedef std::map<std::string, Value *> Sorted;
-            Sorted sorted;
-            for (auto & i : *v.attrs)
-                sorted.emplace(state->symbols[i.name], i.value);
-
-            for (auto & i : sorted) {
-                printAttributeName(str, i.first);
-                str << " = ";
-                if (seen.count(i.second))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *i.second, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << "; ";
-            }
-
-            str << "}";
-        } else
-            str << "{ ... }";
-
-        break;
-    }
-
-    case nList:
-        seen.insert(&v);
-
-        str << "[ ";
-        if (maxDepth > 0)
-            for (auto elem : v.listItems()) {
-                if (seen.count(elem))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *elem, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << " ";
-            }
-        else
-            str << "... ";
-        str << "]";
-        break;
-
-    case nFunction:
-        if (v.isLambda()) {
-            std::ostringstream s;
-            s << state->positions[v.lambda.fun->pos];
-            str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
-        } else if (v.isPrimOp()) {
-            str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
-        } else if (v.isPrimOpApp()) {
-            str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
-        } else {
-            abort();
-        }
-        break;
-
-    case nFloat:
-        str << v.fpoint;
-        break;
-
-    case nThunk:
-    case nExternal:
-    default:
-        str << ANSI_RED "«unknown»" ANSI_NORMAL;
-        break;
-    }
-
-    return str;
 }
 
 

src/libexpr/eval-inline.hh

@@ -73,8 +73,6 @@ Env & EvalState::allocEnv(size_t size)
 #endif
         env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
 
-    env->type = Env::Plain;
-
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
     return *env;
@@ -83,13 +81,6 @@ Env & EvalState::allocEnv(size_t size)
 
 [[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const PosIdx pos)
-{
-    forceValue(v, [&]() { return pos; });
-}
-
-
-template<typename Callable>
-void EvalState::forceValue(Value & v, Callable getPos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
@@ -100,15 +91,12 @@ void EvalState::forceValue(Value & v, Callable getPos)
             expr->eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
+            tryFixupBlackHolePos(v, pos);
             throw;
         }
     }
-    else if (v.isApp()) {
+    else if (v.isApp())
-        PosIdx pos = getPos();
         callFunction(*v.app.left, *v.app.right, v, pos);
-    }
-    else if (v.isBlackhole())
-        error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>();
 }
 
 

src/libexpr/eval-settings.hh

@@ -124,6 +124,9 @@ struct EvalSettings : Config
 
     Setting<bool> traceVerbose{this, false, "trace-verbose",
         "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
+
+    Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
+        "The maximum function call depth to allow before erroring."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/eval.cc

@@ -19,6 +19,7 @@
 #include "signals.hh"
 #include "gc-small-vector.hh"
 #include "url.hh"
+#include "fetch-to-store.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -104,116 +105,23 @@ RootValue allocRootValue(Value * v)
 #endif
 }
 
-void Value::print(const SymbolTable &symbols, std::ostream &str,
-                  std::set<const void *> *seen, int depth) const
-
-{
-    checkInterrupt();
-
-    if (depth <= 0) {
-        str << "«too deep»";
-        return;
-    }
-    switch (internalType) {
-    case tInt:
-        str << integer;
-        break;
-    case tBool:
-        printLiteralBool(str, boolean);
-        break;
-    case tString:
-        printLiteralString(str, string_view());
-        break;
-    case tPath:
-        str << path().to_string(); // !!! escaping?
-        break;
-    case tNull:
-        str << "null";
-        break;
-    case tAttrs: {
-        if (seen && !attrs->empty() && !seen->insert(attrs).second)
-            str << "«repeated»";
-        else {
-            str << "{ ";
-            for (auto & i : attrs->lexicographicOrder(symbols)) {
-                str << symbols[i->name] << " = ";
-                i->value->print(symbols, str, seen, depth - 1);
-                str << "; ";
-            }
-            str << "}";
-        }
-        break;
-    }
-    case tList1:
-    case tList2:
-    case tListN:
-        if (seen && listSize() && !seen->insert(listElems()).second)
-            str << "«repeated»";
-        else {
-            str << "[ ";
-            for (auto v2 : listItems()) {
-                if (v2)
-                    v2->print(symbols, str, seen, depth - 1);
-                else
-                    str << "(nullptr)";
-                str << " ";
-            }
-            str << "]";
-        }
-        break;
-    case tThunk:
-    case tApp:
-        str << "<CODE>";
-        break;
-    case tLambda:
-        str << "<LAMBDA>";
-        break;
-    case tPrimOp:
-        str << "<PRIMOP>";
-        break;
-    case tPrimOpApp:
-        str << "<PRIMOP-APP>";
-        break;
-    case tExternal:
-        str << *external;
-        break;
-    case tFloat:
-        str << fpoint;
-        break;
-    case tBlackhole:
-        // Although we know for sure that it's going to be an infinite recursion
-        // when this value is accessed _in the current context_, it's likely
-        // that the user will misinterpret a simpler «infinite recursion» output
-        // as a definitive statement about the value, while in fact it may be
-        // a valid value after `builtins.trace` and perhaps some other steps
-        // have completed.
-        str << "«potential infinite recursion»";
-        break;
-    default:
-        printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType);
-        abort();
-    }
-}
-
-void Value::print(const SymbolTable &symbols, std::ostream &str,
-                  bool showRepeated, int depth) const {
-    std::set<const void *> seen;
-    print(symbols, str, showRepeated ? nullptr : &seen, depth);
-}
-
 // Pretty print types for assertion errors
 std::ostream & operator << (std::ostream & os, const ValueType t) {
     os << showType(t);
     return os;
 }
 
-std::string printValue(const EvalState & state, const Value & v)
+std::string printValue(EvalState & state, Value & v)
 {
     std::ostringstream out;
-    v.print(state.symbols, out);
+    v.print(state, out);
     return out.str();
 }
 
+void Value::print(EvalState & state, std::ostream & str, PrintOptions options)
+{
+    printValue(state, str, *this, options);
+}
 
 const Value * getPrimOp(const Value &v) {
     const Value * primOp = &v;
@@ -256,9 +164,8 @@ std::string showType(const Value & v)
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tThunk: return "a thunk";
+        case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk";
         case tApp: return "a function application";
-        case tBlackhole: return "a black hole";
     default:
         return std::string(showType(v.type()));
     }
@@ -543,7 +450,7 @@ EvalState::EvalState(
     , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
 #endif
     , baseEnv(allocEnv(128))
-    , staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
+    , staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
 {
     corepkgsFS->setPathDisplay("<nix", ">");
     internalFS->setPathDisplay("«nix-internal»", "");
@@ -554,6 +461,8 @@ EvalState::EvalState(
 
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
 
+    vEmptyList.mkList(0);
+
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
         for (auto & i : _searchPath.elements)
@@ -707,6 +616,26 @@ void PrimOp::check()
 }
 
 
+std::ostream & operator<<(std::ostream & output, PrimOp & primOp)
+{
+    output << "primop " << primOp.name;
+    return output;
+}
+
+
+PrimOp * Value::primOpAppPrimOp() const
+{
+    Value * left = primOpApp.left;
+    while (left && !left->isPrimOp()) {
+        left = left->primOpApp.left;
+    }
+
+    if (!left)
+        return nullptr;
+    return left->primOp;
+}
+
+
 void Value::mkPrimOp(PrimOp * p)
 {
     p->check();
@@ -781,7 +710,7 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se)
 // just for the current level of Env, not the whole chain.
 void printWithBindings(const SymbolTable & st, const Env & env)
 {
-    if (env.type == Env::HasWithAttrs) {
+    if (!env.values[0]->isThunk()) {
         std::cout << "with: ";
         std::cout << ANSI_MAGENTA;
         Bindings::iterator j = env.values[0]->attrs->begin();
@@ -835,7 +764,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
     if (env.up && se.up) {
         mapStaticEnvBindings(st, *se.up, *env.up, vm);
 
-        if (env.type == Env::HasWithAttrs) {
+        if (!env.values[0]->isThunk()) {
             // add 'with' bindings.
             Bindings::iterator j = env.values[0]->attrs->begin();
             while (j != env.values[0]->attrs->end()) {
@@ -868,7 +797,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
         ? std::make_unique<DebugTraceStacker>(
             *this,
             DebugTrace {
-                .pos = error->info().errPos ? error->info().errPos : static_cast<std::shared_ptr<AbstractPos>>(positions[expr.getPos()]),
+                .pos = error->info().errPos ? error->info().errPos : positions[expr.getPos()],
                 .expr = expr,
                 .env = env,
                 .hint = error->info().msg,
@@ -907,7 +836,7 @@ static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
     EvalState & state,
     Expr & expr,
     Env & env,
-    std::shared_ptr<AbstractPos> && pos,
+    std::shared_ptr<Pos> && pos,
     const char * s,
     const std::string & s2)
 {
@@ -973,22 +902,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 
     if (!var.fromWith) return env->values[var.displ];
 
+    // This early exit defeats the `maybeThunk` optimization for variables from `with`,
+    // The added complexity of handling this appears to be similarly in cost, or
+    // the cases where applicable were insignificant in the first place.
+    if (noEval) return nullptr;
+
+    auto * fromWith = var.fromWith;
     while (1) {
-        if (env->type == Env::HasWithExpr) {
+        forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
-            if (noEval) return 0;
-            Value * v = allocValue();
-            evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, "<borked>");
-            env->values[0] = v;
-            env->type = Env::HasWithAttrs;
-        }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
             if (countCalls) attrSelects[j->pos]++;
             return j->value;
         }
-        if (!env->prevWith)
+        if (!fromWith->parentWith)
             error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow<UndefinedVarError>();
-        for (size_t l = env->prevWith; l; --l, env = env->up) ;
+        for (size_t l = fromWith->prevWith; l; --l, env = env->up) ;
+        fromWith = fromWith->parentWith;
     }
 }
 
@@ -1184,7 +1114,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
                 *this,
                 *e,
                 this->baseEnv,
-                e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr,
+                e->getPos() ? std::make_shared<Pos>(positions[e->getPos()]) : nullptr,
                 "while evaluating the file '%1%':", resolvedPath.to_string())
             : nullptr;
 
@@ -1384,6 +1314,15 @@ void ExprList::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+Value * ExprList::maybeThunk(EvalState & state, Env & env)
+{
+    if (elems.empty()) {
+        return &state.vEmptyList;
+    }
+    return Expr::maybeThunk(state, env);
+}
+
+
 void ExprVar::eval(EvalState & state, Env & env, Value & v)
 {
     Value * v2 = state.lookupVar(&env, *this, false);
@@ -1505,9 +1444,27 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
     v.mkLambda(&env, this);
 }
 
+namespace {
+/** Increments a count on construction and decrements on destruction.
+ */
+class CallDepth {
+  size_t & count;
+public:
+  CallDepth(size_t & count) : count(count) {
+    ++count;
+  }
+  ~CallDepth() {
+    --count;
+  }
+};
+};
 
 void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
 {
+    if (callDepth > evalSettings.maxCallDepth)
+        error("stack overflow; max-call-depth exceeded").atPos(pos).template debugThrow<EvalError>();
+    CallDepth _level(callDepth);
+
     auto trace = evalSettings.traceFunctionCalls
         ? std::make_unique<FunctionCallTrace>(positions[pos])
         : nullptr;
@@ -1646,15 +1603,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 return;
             } else {
                 /* We have all the arguments, so call the primop. */
-                auto name = vCur.primOp->name;
+                auto * fn = vCur.primOp;
 
                 nrPrimOpCalls++;
-                if (countCalls) primOpCalls[name]++;
+                if (countCalls) primOpCalls[fn->name]++;
 
                 try {
-                    vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur);
+                    fn->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
+                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1691,18 +1648,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 for (size_t i = 0; i < argsLeft; ++i)
                     vArgs[argsDone + i] = args[i];
 
-                auto name = primOp->primOp->name;
+                auto fn = primOp->primOp;
                 nrPrimOpCalls++;
-                if (countCalls) primOpCalls[name]++;
+                if (countCalls) primOpCalls[fn->name]++;
 
                 try {
                     // TODO:
                     // 1. Unify this and above code. Heavily redundant.
                     // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc)
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
-                    primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
+                    fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
+                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1816,9 +1773,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 {
     Env & env2(state.allocEnv(1));
     env2.up = &env;
-    env2.prevWith = prevWith;
+    env2.values[0] = attrs->maybeThunk(state, env);
-    env2.type = Env::HasWithExpr;
-    env2.values[0] = (Value *) attrs;
 
     body->eval(state, env2, v);
 }
@@ -2056,6 +2011,29 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
+{
+    state.error("infinite recursion encountered")
+        .debugThrow<InfiniteRecursionError>();
+}
+
+// always force this to be separate, otherwise forceValue may inline it and take
+// a massive perf hit
+[[gnu::noinline]]
+void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
+{
+    if (!v.isBlackhole())
+        return;
+    auto e = std::current_exception();
+    try {
+        std::rethrow_exception(e);
+    } catch (InfiniteRecursionError & e) {
+        e.err.errPos = positions[pos];
+    } catch (...) {
+    }
+}
+
+
 void EvalState::forceValueDeep(Value & v)
 {
     std::set<const Value *> seen;
@@ -2065,7 +2043,7 @@ void EvalState::forceValueDeep(Value & v)
     recurse = [&](Value & v) {
         if (!seen.insert(&v).second) return;
 
-        forceValue(v, [&]() { return v.determinePos(noPos); });
+        forceValue(v, v.determinePos(noPos));
 
         if (v.type() == nAttrs) {
             for (auto & i : *v.attrs)
@@ -2317,7 +2295,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = fetchToStore(*store, path, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@@ -2457,7 +2435,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             return v1.boolean == v2.boolean;
 
         case nString:
-            return v1.string_view().compare(v2.string_view()) == 0;
+            return strcmp(v1.c_str(), v2.c_str()) == 0;
 
         case nPath:
             return

src/libexpr/eval.hh

@@ -84,6 +84,8 @@ struct PrimOp
     void check();
 };
 
+std::ostream & operator<<(std::ostream & output, PrimOp & primOp);
+
 /**
  * Info about a constant
  */
@@ -116,11 +118,6 @@ struct Constant
 struct Env
 {
     Env * up;
-    /**
-     * Number of of levels up to next `with` environment
-     */
-    unsigned short prevWith:14;
-    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
@@ -132,7 +129,7 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati
 void copyContext(const Value & v, NixStringContext & context);
 
 
-std::string printValue(const EvalState & state, const Value & v);
+std::string printValue(EvalState & state, Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
@@ -147,7 +144,7 @@ struct RegexCache;
 std::shared_ptr<RegexCache> makeRegexCache();
 
 struct DebugTrace {
-    std::shared_ptr<AbstractPos> pos;
+    std::shared_ptr<Pos> pos;
     const Expr & expr;
     const Env & env;
     hintformat hint;
@@ -218,6 +215,11 @@ public:
 
     Bindings emptyBindings;
 
+    /**
+     * Empty list constant.
+     */
+    Value vEmptyList;
+
     /**
      * The accessor for the root filesystem.
      */
@@ -460,8 +462,7 @@ public:
      */
     inline void forceValue(Value & v, const PosIdx pos);
 
-    template <typename Callable>
+    void tryFixupBlackHolePos(Value & v, PosIdx pos);
-    inline void forceValue(Value & v, Callable getPos);
 
     /**
      * Force a value, then recursively force list elements and
@@ -623,6 +624,11 @@ private:
         const SourcePath & basePath,
         std::shared_ptr<StaticEnv> & staticEnv);
 
+    /**
+     * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack.
+     */
+    size_t callDepth = 0;
+
 public:
 
     /**

src/libexpr/flake/url-name.cc

@@ -4,7 +4,7 @@
 
 namespace nix {
 
-static const std::string attributeNamePattern("[a-z0-9_-]+");
+static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
 static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
 static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
 static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");

src/libexpr/get-drvs.cc

@@ -198,7 +198,7 @@ StringSet DrvInfo::queryMetaNames()
 
 bool DrvInfo::checkMeta(Value & v)
 {
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
     if (v.type() == nList) {
         for (auto elem : v.listItems())
             if (!checkMeta(*elem)) return false;
@@ -304,7 +304,7 @@ static bool getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     try {
-        state.forceValue(v, [&]() { return v.determinePos(noPos); });
+        state.forceValue(v, v.determinePos(noPos));
         if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =

src/libexpr/lexer.l

@@ -1,4 +1,5 @@
 %option reentrant bison-bridge bison-locations
+%option align
 %option noyywrap
 %option never-interactive
 %option stack
@@ -35,9 +36,6 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 #define CUR_POS makeCurPos(*yylloc, data)
 
-// backup to recover from yyless(0)
-thread_local YYLTYPE prev_yylloc;
-
 static void initLoc(YYLTYPE * loc)
 {
     loc->first_line = loc->last_line = 1;
@@ -46,7 +44,7 @@ static void initLoc(YYLTYPE * loc)
 
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
-    prev_yylloc = *loc;
+    loc->stash();
 
     loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
@@ -230,7 +228,7 @@ or          { return OR_KW; }
 {HPATH_START}\$\{ {
   PUSH_STATE(PATH_START);
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
 }
 
 <PATH_START>{PATH_SEG} {
@@ -286,7 +284,7 @@ or          { return OR_KW; }
      context (it may be ')', ';', or something of that sort) */
   POP_STATE();
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
   return PATH_END;
 }
 

src/libexpr/local.mk

@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -pthread
+libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
   libexpr_LDFLAGS += -ldl
 endif

src/libexpr/nixexpr.cc

@@ -9,57 +9,7 @@
 
 namespace nix {
 
-struct PosAdapter : AbstractPos
+ExprBlackHole eBlackHole;
-{
-    Pos::Origin origin;
-
-    PosAdapter(Pos::Origin origin)
-        : origin(std::move(origin))
-    {
-    }
-
-    std::optional<std::string> getSource() const override
-    {
-        return std::visit(overloaded {
-            [](const Pos::none_tag &) -> std::optional<std::string> {
-                return std::nullopt;
-            },
-            [](const Pos::Stdin & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const Pos::String & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const SourcePath & path) -> std::optional<std::string> {
-                try {
-                    return path.readFile();
-                } catch (Error &) {
-                    return std::nullopt;
-                }
-            }
-        }, origin);
-    }
-
-    void print(std::ostream & out) const override
-    {
-        std::visit(overloaded {
-            [&](const Pos::none_tag &) { out << "«none»"; },
-            [&](const Pos::Stdin &) { out << "«stdin»"; },
-            [&](const Pos::String & s) { out << "«string»"; },
-            [&](const SourcePath & path) { out << path; }
-        }, origin);
-    }
-};
-
-Pos::operator std::shared_ptr<AbstractPos>() const
-{
-    auto pos = std::make_shared<PosAdapter>(origin);
-    pos->line = line;
-    pos->column = column;
-    return pos;
-}
 
 // FIXME: remove, because *symbols* are abstract and do not have a single
 //        textual representation; see printIdentifier()
@@ -266,17 +216,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
 }
 
 
-std::ostream & operator << (std::ostream & str, const Pos & pos)
-{
-    if (auto pos2 = (std::shared_ptr<AbstractPos>) pos) {
-        str << *pos2;
-    } else
-        str << "undefined position";
-
-    return str;
-}
-
-
 std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 {
     std::ostringstream out;
@@ -331,6 +270,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    fromWith = nullptr;
+
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
     const StaticEnv * curEnv;
@@ -342,7 +283,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         } else {
             auto i = curEnv->find(name);
             if (i != curEnv->vars.end()) {
-                fromWith = false;
                 this->level = level;
                 displ = i->second;
                 return;
@@ -358,7 +298,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
             .msg = hintfmt("undefined variable '%1%'", es.symbols[name]),
             .errPos = es.positions[pos]
         });
-    fromWith = true;
+    for (auto * e = env.get(); e && !fromWith; e = e->up)
+        fromWith = e->isWith;
     this->level = withLevel;
 }
 
@@ -391,7 +332,7 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
-        auto newEnv = std::make_shared<StaticEnv>(false, env.get(), recursive ? attrs.size() : 0);
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), recursive ? attrs.size() : 0);
 
         Displacement displ = 0;
         for (auto & i : attrs)
@@ -433,7 +374,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     auto newEnv = std::make_shared<StaticEnv>(
-        false, env.get(),
+        nullptr, env.get(),
         (hasFormals() ? formals->formals.size() : 0) +
         (!arg ? 0 : 1));
 
@@ -469,7 +410,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
-    auto newEnv = std::make_shared<StaticEnv>(false, env.get(), attrs->attrs.size());
+    auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
 
     Displacement displ = 0;
     for (auto & i : attrs->attrs)
@@ -488,6 +429,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    parentWith = nullptr;
+    for (auto * e = env.get(); e && !parentWith; e = e->up)
+        parentWith = e->isWith;
+
     /* Does this `with' have an enclosing `with'?  If so, record its
        level so that `lookupVar' can look up variables in the previous
        `with' if this one doesn't contain the desired attribute. */
@@ -504,7 +449,7 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         es.exprEnvs.insert(std::make_pair(this, env));
 
     attrs->bindVars(es, env);
-    auto newEnv = std::make_shared<StaticEnv>(true, env.get());
+    auto newEnv = std::make_shared<StaticEnv>(this, env.get());
     body->bindVars(es, newEnv);
 }
 

src/libexpr/nixexpr.hh

@@ -8,6 +8,7 @@
 #include "symbol-table.hh"
 #include "error.hh"
 #include "chunked-vector.hh"
+#include "position.hh"
 
 namespace nix {
 
@@ -21,25 +22,11 @@ MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
 MakeError(MissingArgumentError, EvalError);
 
-/**
+class InfiniteRecursionError : public EvalError
- * Position objects.
- */
-struct Pos
 {
-    uint32_t line;
+    friend class EvalState;
-    uint32_t column;
+public:
-
+    using EvalError::EvalError;
-    struct none_tag { };
-    struct Stdin { ref<std::string> source; };
-    struct String { ref<std::string> source; };
-
-    typedef std::variant<none_tag, Stdin, String, SourcePath> Origin;
-
-    Origin origin;
-
-    explicit operator bool() const { return line > 0; }
-
-    operator std::shared_ptr<AbstractPos>() const;
 };
 
 class PosIdx {
@@ -74,7 +61,7 @@ public:
         mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
 
         // Used for searching in PosTable::[].
-        explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {}
+        explicit Origin(uint32_t idx): idx(idx), origin{std::monostate()} {}
 
     public:
         const Pos::Origin origin;
@@ -125,12 +112,11 @@ public:
 
 inline PosIdx noPos = {};
 
-std::ostream & operator << (std::ostream & str, const Pos & pos);
-
 
 struct Env;
 struct Value;
 class EvalState;
+struct ExprWith;
 struct StaticEnv;
 
 
@@ -219,8 +205,11 @@ struct ExprVar : Expr
     Symbol name;
 
     /* Whether the variable comes from an environment (e.g. a rec, let
-       or function argument) or from a "with". */
+       or function argument) or from a "with".
-    bool fromWith;
+
+       `nullptr`: Not from a `with`.
+       Valid pointer: the nearest, innermost `with` expression to query first. */
+    ExprWith * fromWith;
 
     /* In the former case, the value is obtained by going `level`
        levels up from the current environment and getting the
@@ -292,6 +281,7 @@ struct ExprList : Expr
     std::vector<Expr *> elems;
     ExprList() { };
     COMMON_METHODS
+    Value * maybeThunk(EvalState & state, Env & env) override;
 
     PosIdx getPos() const override
     {
@@ -378,6 +368,7 @@ struct ExprWith : Expr
     PosIdx pos;
     Expr * attrs, * body;
     size_t prevWith;
+    ExprWith * parentWith;
     ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -455,20 +446,30 @@ struct ExprPos : Expr
     COMMON_METHODS
 };
 
+/* only used to mark thunks as black holes. */
+struct ExprBlackHole : Expr
+{
+    void show(const SymbolTable & symbols, std::ostream & str) const override {}
+    void eval(EvalState & state, Env & env, Value & v) override;
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override {}
+};
+
+extern ExprBlackHole eBlackHole;
+
 
 /* Static environments are used to map variable names onto (level,
    displacement) pairs used to obtain the value of the variable at
    runtime. */
 struct StaticEnv
 {
-    bool isWith;
+    ExprWith * isWith;
     const StaticEnv * up;
 
     // Note: these must be in sorted order.
     typedef std::vector<std::pair<Symbol, Displacement>> Vars;
     Vars vars;
 
-    StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
+    StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
         vars.reserve(expectedSize);
     };
 

src/libexpr/parser.y

@@ -28,6 +28,31 @@
 
 namespace nix {
 
+#define YYLTYPE ::nix::ParserLocation
+    struct ParserLocation
+    {
+        int first_line, first_column;
+        int last_line, last_column;
+
+        // backup to recover from yyless(0)
+        int stashed_first_line, stashed_first_column;
+        int stashed_last_line, stashed_last_column;
+
+        void stash() {
+          stashed_first_line = first_line;
+          stashed_first_column = first_column;
+          stashed_last_line = last_line;
+          stashed_last_column = last_column;
+        }
+
+        void unstash() {
+          first_line = stashed_first_line;
+          first_column = stashed_first_column;
+          last_line = stashed_last_line;
+          last_column = stashed_last_column;
+        }
+    };
+
     struct ParseData
     {
         EvalState & state;

src/libexpr/primops.cc

@@ -16,6 +16,7 @@
 #include "value-to-xml.hh"
 #include "primops.hh"
 #include "fs-input-accessor.hh"
+#include "fetch-to-store.hh"
 
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
@@ -84,14 +85,14 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     /* Build/substitute the context. */
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->buildPaths(buildReqs);
+    buildStore->buildPaths(buildReqs, bmNormal, store);
+
+    StorePathSet outputsToCopyAndAllow;
 
     for (auto & drv : drvs) {
-        auto outputs = resolveDerivedPath(*store, drv);
+        auto outputs = resolveDerivedPath(*buildStore, drv, &*store);
         for (auto & [outputName, outputPath] : outputs) {
-            /* Add the output of this derivations to the allowed
+            outputsToCopyAndAllow.insert(outputPath);
-               paths. */
-            allowPath(store->toRealPath(outputPath));
 
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -101,12 +102,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                             .drvPath = drv.drvPath,
                             .output = outputName,
                         }).render(),
-                    store->printStorePath(outputPath)
+                    buildStore->printStorePath(outputPath)
                 );
             }
         }
     }
 
+    if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
+    for (auto & outputPath : outputsToCopyAndAllow) {
+        /* Add the output of this derivations to the allowed
+           paths. */
+        allowPath(store->toRealPath(outputPath));
+    }
+
     return res;
 }
 
@@ -214,7 +222,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
             Env * env = &state.allocEnv(vScope->attrs->size());
             env->up = &state.baseEnv;
 
-            auto staticEnv = std::make_shared<StaticEnv>(false, state.staticBaseEnv.get(), vScope->attrs->size());
+            auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv.get(), vScope->attrs->size());
 
             unsigned int displ = 0;
             for (auto & attr : *vScope->attrs) {
@@ -584,7 +592,7 @@ struct CompareValues
                 case nFloat:
                     return v1->fpoint < v2->fpoint;
                 case nString:
-                    return v1->string_view().compare(v2->string_view()) < 0;
+                    return strcmp(v1->c_str(), v2->c_str()) < 0;
                 case nPath:
                     // Note: we don't take the accessor into account
                     // since it's not obvious how to compare them in a
@@ -2233,7 +2241,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = path.fetchToStore(*state.store, name, method, filter.get(), state.repair);
+            auto dstPath = fetchToStore(*state.store, path, name, method, filter.get(), state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);
@@ -2405,7 +2413,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args,
         (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]);
 
     std::sort(v.listElems(), v.listElems() + n,
-              [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; });
+              [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; });
 }
 
 static RegisterPrimOp primop_attrNames({

src/libexpr/print-ambiguous.cc

@@ -0,0 +1,100 @@
+#include "print-ambiguous.hh"
+#include "print.hh"
+#include "signals.hh"
+
+namespace nix {
+
+// See: https://github.com/NixOS/nix/issues/9730
+void printAmbiguous(
+    Value &v,
+    const SymbolTable &symbols,
+    std::ostream &str,
+    std::set<const void *> *seen,
+    int depth)
+{
+    checkInterrupt();
+
+    if (depth <= 0) {
+        str << "«too deep»";
+        return;
+    }
+    switch (v.type()) {
+    case nInt:
+        str << v.integer;
+        break;
+    case nBool:
+        printLiteralBool(str, v.boolean);
+        break;
+    case nString:
+        printLiteralString(str, v.string_view());
+        break;
+    case nPath:
+        str << v.path().to_string(); // !!! escaping?
+        break;
+    case nNull:
+        str << "null";
+        break;
+    case nAttrs: {
+        if (seen && !v.attrs->empty() && !seen->insert(v.attrs).second)
+            str << "«repeated»";
+        else {
+            str << "{ ";
+            for (auto & i : v.attrs->lexicographicOrder(symbols)) {
+                str << symbols[i->name] << " = ";
+                printAmbiguous(*i->value, symbols, str, seen, depth - 1);
+                str << "; ";
+            }
+            str << "}";
+        }
+        break;
+    }
+    case nList:
+        if (seen && v.listSize() && !seen->insert(v.listElems()).second)
+            str << "«repeated»";
+        else {
+            str << "[ ";
+            for (auto v2 : v.listItems()) {
+                if (v2)
+                    printAmbiguous(*v2, symbols, str, seen, depth - 1);
+                else
+                    str << "(nullptr)";
+                str << " ";
+            }
+            str << "]";
+        }
+        break;
+    case nThunk:
+        if (!v.isBlackhole()) {
+            str << "<CODE>";
+        } else {
+            // Although we know for sure that it's going to be an infinite recursion
+            // when this value is accessed _in the current context_, it's likely
+            // that the user will misinterpret a simpler «infinite recursion» output
+            // as a definitive statement about the value, while in fact it may be
+            // a valid value after `builtins.trace` and perhaps some other steps
+            // have completed.
+            str << "«potential infinite recursion»";
+        }
+        break;
+    case nFunction:
+        if (v.isLambda()) {
+            str << "<LAMBDA>";
+        } else if (v.isPrimOp()) {
+            str << "<PRIMOP>";
+        } else if (v.isPrimOpApp()) {
+            str << "<PRIMOP-APP>";
+        }
+        break;
+    case nExternal:
+        str << *v.external;
+        break;
+    case nFloat:
+        str << v.fpoint;
+        break;
+    default:
+        printError("Nix evaluator internal error: printAmbiguous: invalid value type");
+        abort();
+    }
+}
+
+}

src/libexpr/print-ambiguous.hh

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "value.hh"
+
+namespace nix {
+
+/**
+ * Print a value in the deprecated format used by `nix-instantiate --eval` and
+ * `nix-env` (for manifests).
+ *
+ * This output can't be changed because it's part of the `nix-instantiate` API,
+ * but it produces ambiguous output; unevaluated thunks and lambdas (and a few
+ * other types) are printed as Nix path syntax like `<CODE>`.
+ *
+ * See: https://github.com/NixOS/nix/issues/9730
+ */
+void printAmbiguous(
+    Value &v,
+    const SymbolTable &symbols,
+    std::ostream &str,
+    std::set<const void *> *seen,
+    int depth);
+
+}

src/libexpr/print-options.hh

@@ -0,0 +1,52 @@
+#pragma once
+/**
+ * @file
+ * @brief Options for printing Nix values.
+ */
+
+#include <limits>
+
+namespace nix {
+
+/**
+ * Options for printing Nix values.
+ */
+struct PrintOptions
+{
+    /**
+     * If true, output ANSI color sequences.
+     */
+    bool ansiColors = false;
+    /**
+     * If true, force values.
+     */
+    bool force = false;
+    /**
+     * If true and `force` is set, print derivations as
+     * `«derivation /nix/store/...»` instead of as attribute sets.
+     */
+    bool derivationPaths = false;
+    /**
+     * If true, track which values have been printed and skip them on
+     * subsequent encounters. Useful for self-referential values.
+     */
+    bool trackRepeated = true;
+    /**
+     * Maximum depth to evaluate to.
+     */
+    size_t maxDepth = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum number of attributes in an attribute set to print.
+     */
+    size_t maxAttrs = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum number of list items to print.
+     */
+    size_t maxListItems = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum string length to print.
+     */
+    size_t maxStringLength = std::numeric_limits<size_t>::max();
+};
+
+}

src/libexpr/print.cc

@@ -1,24 +1,66 @@
-#include "print.hh"
+#include <limits>
 #include <unordered_set>
 
+#include "print.hh"
+#include "ansicolor.hh"
+#include "signals.hh"
+#include "store-api.hh"
+#include "terminal.hh"
+#include "english.hh"
+
 namespace nix {
 
-std::ostream &
+void printElided(
-printLiteralString(std::ostream & str, const std::string_view string)
+    std::ostream & output,
+    unsigned int value,
+    const std::string_view single,
+    const std::string_view plural,
+    bool ansiColors)
 {
+    if (ansiColors)
+        output << ANSI_FAINT;
+    output << " «";
+    pluralize(output, value, single, plural);
+    output << " elided»";
+    if (ansiColors)
+        output << ANSI_NORMAL;
+}
+
+
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string, size_t maxLength, bool ansiColors)
+{
+    size_t charsPrinted = 0;
+    if (ansiColors)
+        str << ANSI_MAGENTA;
     str << "\"";
     for (auto i = string.begin(); i != string.end(); ++i) {
+        if (charsPrinted >= maxLength) {
+            str << "\"";
+            printElided(str, string.length() - charsPrinted, "byte", "bytes", ansiColors);
+            return str;
+        }
+
         if (*i == '\"' || *i == '\\') str << "\\" << *i;
         else if (*i == '\n') str << "\\n";
         else if (*i == '\r') str << "\\r";
         else if (*i == '\t') str << "\\t";
         else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
         else str << *i;
+        charsPrinted++;
     }
     str << "\"";
+    if (ansiColors)
+        str << ANSI_NORMAL;
     return str;
 }
 
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string)
+{
+    return printLiteralString(str, string, std::numeric_limits<size_t>::max(), false);
+}
+
 std::ostream &
 printLiteralBool(std::ostream & str, bool boolean)
 {
@@ -90,5 +132,373 @@ printAttributeName(std::ostream & str, std::string_view name) {
     return str;
 }
 
+bool isImportantAttrName(const std::string& attrName)
+{
+    return attrName == "type" || attrName == "_type";
+}
+
+typedef std::pair<std::string, Value *> AttrPair;
+
+struct ImportantFirstAttrNameCmp
+{
+
+    bool operator()(const AttrPair& lhs, const AttrPair& rhs) const
+    {
+        auto lhsIsImportant = isImportantAttrName(lhs.first);
+        auto rhsIsImportant = isImportantAttrName(rhs.first);
+        return std::forward_as_tuple(!lhsIsImportant, lhs.first)
+            < std::forward_as_tuple(!rhsIsImportant, rhs.first);
+    }
+};
+
+typedef std::set<Value *> ValuesSeen;
+
+class Printer
+{
+private:
+    std::ostream & output;
+    EvalState & state;
+    PrintOptions options;
+    std::optional<ValuesSeen> seen;
+
+    void printRepeated()
+    {
+        if (options.ansiColors)
+            output << ANSI_MAGENTA;
+        output << "«repeated»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printNullptr()
+    {
+        if (options.ansiColors)
+            output << ANSI_MAGENTA;
+        output << "«nullptr»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printElided(unsigned int value, const std::string_view single, const std::string_view plural)
+    {
+        ::nix::printElided(output, value, single, plural, options.ansiColors);
+    }
+
+    void printInt(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << v.integer;
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printFloat(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << v.fpoint;
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printBool(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        printLiteralBool(output, v.boolean);
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printString(Value & v)
+    {
+        printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors);
+    }
+
+    void printPath(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_GREEN;
+        output << v.path().to_string(); // !!! escaping?
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printNull()
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << "null";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printDerivation(Value & v)
+    {
+        try {
+            Bindings::iterator i = v.attrs->find(state.sDrvPath);
+            NixStringContext context;
+            std::string storePath;
+            if (i != v.attrs->end())
+                storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
+
+            if (options.ansiColors)
+                output << ANSI_GREEN;
+            output << "«derivation";
+            if (!storePath.empty()) {
+                output << " " << storePath;
+            }
+            output << "»";
+            if (options.ansiColors)
+                output << ANSI_NORMAL;
+        } catch (BaseError & e) {
+            printError_(e);
+        }
+    }
+
+    void printAttrs(Value & v, size_t depth)
+    {
+        if (seen && !seen->insert(&v).second) {
+            printRepeated();
+            return;
+        }
+
+        if (options.force && options.derivationPaths && state.isDerivation(v)) {
+            printDerivation(v);
+        } else if (depth < options.maxDepth) {
+            output << "{ ";
+
+            std::vector<std::pair<std::string, Value *>> sorted;
+            for (auto & i : *v.attrs)
+                sorted.emplace_back(std::pair(state.symbols[i.name], i.value));
+
+            if (options.maxAttrs == std::numeric_limits<size_t>::max())
+                std::sort(sorted.begin(), sorted.end());
+            else
+                std::sort(sorted.begin(), sorted.end(), ImportantFirstAttrNameCmp());
+
+            size_t attrsPrinted = 0;
+            for (auto & i : sorted) {
+                if (attrsPrinted >= options.maxAttrs) {
+                    printElided(sorted.size() - attrsPrinted, "attribute", "attributes");
+                    break;
+                }
+
+                printAttributeName(output, i.first);
+                output << " = ";
+                print(*i.second, depth + 1);
+                output << "; ";
+                attrsPrinted++;
+            }
+
+            output << "}";
+        } else
+            output << "{ ... }";
+    }
+
+    void printList(Value & v, size_t depth)
+    {
+        if (seen && v.listSize() && !seen->insert(&v).second) {
+            printRepeated();
+            return;
+        }
+
+        output << "[ ";
+        if (depth < options.maxDepth) {
+            size_t listItemsPrinted = 0;
+            for (auto elem : v.listItems()) {
+                if (listItemsPrinted >= options.maxListItems) {
+                    printElided(v.listSize() - listItemsPrinted, "item", "items");
+                    break;
+                }
+
+                if (elem) {
+                    print(*elem, depth + 1);
+                } else {
+                    printNullptr();
+                }
+                output << " ";
+                listItemsPrinted++;
+            }
+        }
+        else
+            output << "... ";
+        output << "]";
+    }
+
+    void printFunction(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_BLUE;
+        output << "«";
+
+        if (v.isLambda()) {
+            output << "lambda";
+            if (v.lambda.fun) {
+                if (v.lambda.fun->name) {
+                    output << " " << state.symbols[v.lambda.fun->name];
+                }
+
+                std::ostringstream s;
+                s << state.positions[v.lambda.fun->pos];
+                output << " @ " << filterANSIEscapes(s.str());
+            }
+        } else if (v.isPrimOp()) {
+            if (v.primOp)
+                output << *v.primOp;
+            else
+                output << "primop";
+        } else if (v.isPrimOpApp()) {
+            output << "partially applied ";
+            auto primOp = v.primOpAppPrimOp();
+            if (primOp)
+                output << *primOp;
+            else
+                output << "primop";
+        } else {
+            abort();
+        }
+
+        output << "»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printThunk(Value & v)
+    {
+        if (v.isBlackhole()) {
+            // Although we know for sure that it's going to be an infinite recursion
+            // when this value is accessed _in the current context_, it's likely
+            // that the user will misinterpret a simpler «infinite recursion» output
+            // as a definitive statement about the value, while in fact it may be
+            // a valid value after `builtins.trace` and perhaps some other steps
+            // have completed.
+            if (options.ansiColors)
+                output << ANSI_RED;
+            output << "«potential infinite recursion»";
+            if (options.ansiColors)
+                output << ANSI_NORMAL;
+        } else if (v.isThunk() || v.isApp()) {
+            if (options.ansiColors)
+                    output << ANSI_MAGENTA;
+            output << "«thunk»";
+            if (options.ansiColors)
+                    output << ANSI_NORMAL;
+        } else {
+            abort();
+        }
+    }
+
+    void printExternal(Value & v)
+    {
+        v.external->print(output);
+    }
+
+    void printUnknown()
+    {
+        if (options.ansiColors)
+            output << ANSI_RED;
+        output << "«unknown»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printError_(BaseError & e)
+    {
+        if (options.ansiColors)
+            output << ANSI_RED;
+        output << "«" << e.msg() << "»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void print(Value & v, size_t depth)
+    {
+        output.flush();
+        checkInterrupt();
+
+        if (options.force) {
+            try {
+                state.forceValue(v, v.determinePos(noPos));
+            } catch (BaseError & e) {
+                printError_(e);
+                return;
+            }
+        }
+
+        switch (v.type()) {
+
+        case nInt:
+            printInt(v);
+            break;
+
+        case nFloat:
+            printFloat(v);
+            break;
+
+        case nBool:
+            printBool(v);
+            break;
+
+        case nString:
+            printString(v);
+            break;
+
+        case nPath:
+            printPath(v);
+            break;
+
+        case nNull:
+            printNull();
+            break;
+
+        case nAttrs:
+            printAttrs(v, depth);
+            break;
+
+        case nList:
+            printList(v, depth);
+            break;
+
+        case nFunction:
+            printFunction(v);
+            break;
+
+        case nThunk:
+            printThunk(v);
+            break;
+
+        case nExternal:
+            printExternal(v);
+            break;
+
+        default:
+            printUnknown();
+            break;
+        }
+    }
+
+public:
+    Printer(std::ostream & output, EvalState & state, PrintOptions options)
+        : output(output), state(state), options(options) { }
+
+    void print(Value & v)
+    {
+        if (options.trackRepeated) {
+            seen.emplace();
+        } else {
+            seen.reset();
+        }
+
+        ValuesSeen seen;
+        print(v, 0);
+    }
+};
+
+void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options)
+{
+    Printer(output, state, options).print(v);
+}
 
 }

src/libexpr/print.hh

@@ -9,39 +9,44 @@
 
 #include <iostream>
 
+#include "eval.hh"
+#include "print-options.hh"
+
 namespace nix {
-    /**
+
+/**
  * Print a string as a Nix string literal.
  *
  * Quotes and fairly minimal escaping are added.
  *
+ * @param o The output stream to print to
  * @param s The logical string
  */
-    std::ostream & printLiteralString(std::ostream & o, std::string_view s);
+std::ostream & printLiteralString(std::ostream & o, std::string_view s);
-    inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
+inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
     return printLiteralString(o, std::string_view(s));
-    }
+}
-    inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
+inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
     return printLiteralString(o, std::string_view(s));
-    }
+}
 
-    /** Print `true` or `false`. */
+/** Print `true` or `false`. */
-    std::ostream & printLiteralBool(std::ostream & o, bool b);
+std::ostream & printLiteralBool(std::ostream & o, bool b);
 
-    /**
+/**
  * Print a string as an attribute name in the Nix expression language syntax.
  *
  * Prints a quoted string if necessary.
  */
-    std::ostream & printAttributeName(std::ostream & o, std::string_view s);
+std::ostream & printAttributeName(std::ostream & o, std::string_view s);
 
-    /**
+/**
  * Returns `true' is a string is a reserved keyword which requires quotation
  * when printing attribute set field names.
  */
-    bool isReservedKeyword(const std::string_view str);
+bool isReservedKeyword(const std::string_view str);
 
-    /**
+/**
  * Print a string as an identifier in the Nix expression language syntax.
  *
  * FIXME: "identifier" is ambiguous. Identifiers do not have a single
@@ -50,5 +55,8 @@ namespace nix {
  *        expression, or something else entirely, like JSON. Use one of the
  *        `print*` functions instead.
  */
-    std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+
+void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions {});
+
 }

src/libexpr/value.hh

@@ -8,6 +8,8 @@
 #include "symbol-table.hh"
 #include "value/context.hh"
 #include "input-accessor.hh"
+#include "source-path.hh"
+#include "print-options.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -32,7 +34,6 @@ typedef enum {
     tThunk,
     tApp,
     tLambda,
-    tBlackhole,
     tPrimOp,
     tPrimOpApp,
     tExternal,
@@ -62,6 +63,7 @@ class Bindings;
 struct Env;
 struct Expr;
 struct ExprLambda;
+struct ExprBlackHole;
 struct PrimOp;
 class Symbol;
 class PosIdx;
@@ -69,7 +71,7 @@ struct Pos;
 class StorePath;
 class EvalState;
 class XMLWriter;
-
+class Printer;
 
 typedef int64_t NixInt;
 typedef double NixFloat;
@@ -81,6 +83,7 @@ typedef double NixFloat;
 class ExternalValueBase
 {
     friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
+    friend class Printer;
     protected:
     /**
      * Print out the value
@@ -138,11 +141,9 @@ private:
 
     friend std::string showType(const Value & v);
 
-    void print(const SymbolTable &symbols, std::ostream &str, std::set<const void *> *seen, int depth) const;
-
 public:
 
-    void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const;
+    void print(EvalState &state, std::ostream &str, PrintOptions options = PrintOptions {});
 
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
@@ -151,7 +152,7 @@ public:
     // type() == nThunk
     inline bool isThunk() const { return internalType == tThunk; };
     inline bool isApp() const { return internalType == tApp; };
-    inline bool isBlackhole() const { return internalType == tBlackhole; };
+    inline bool isBlackhole() const;
 
     // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
@@ -248,7 +249,7 @@ public:
             case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
             case tExternal: return nExternal;
             case tFloat: return nFloat;
-            case tThunk: case tApp: case tBlackhole: return nThunk;
+            case tThunk: case tApp: return nThunk;
         }
         if (invalidIsThunk)
             return nThunk;
@@ -356,21 +357,22 @@ public:
         lambda.fun = f;
     }
 
-    inline void mkBlackhole()
+    inline void mkBlackhole();
-    {
-        internalType = tBlackhole;
-        // Value will be overridden anyways
-    }
 
     void mkPrimOp(PrimOp * p);
 
     inline void mkPrimOpApp(Value * l, Value * r)
     {
         internalType = tPrimOpApp;
-        app.left = l;
+        primOpApp.left = l;
-        app.right = r;
+        primOpApp.right = r;
     }
 
+    /**
+     * For a `tPrimOpApp` value, get the original `PrimOp` value.
+     */
+    PrimOp * primOpAppPrimOp() const;
+
     inline void mkExternal(ExternalValueBase * e)
     {
         clearValue();
@@ -447,6 +449,20 @@ public:
 };
 
 
+extern ExprBlackHole eBlackHole;
+
+bool Value::isBlackhole() const
+{
+    return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole;
+}
+
+void Value::mkBlackhole()
+{
+    internalType = tThunk;
+    thunk.expr = (Expr*) &eBlackHole;
+}
+
+
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, traceable_allocator<Value *>> ValueVector;
 typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;

src/libfetchers/fetch-to-store.cc

@@ -0,0 +1,68 @@
+#include "fetch-to-store.hh"
+#include "fetchers.hh"
+#include "cache.hh"
+
+namespace nix {
+
+StorePath fetchToStore(
+    Store & store,
+    const SourcePath & path,
+    std::string_view name,
+    ContentAddressMethod method,
+    PathFilter * filter,
+    RepairFlag repair)
+{
+    // FIXME: add an optimisation for the case where the accessor is
+    // an FSInputAccessor pointing to a store path.
+
+    std::optional<fetchers::Attrs> cacheKey;
+
+    if (!filter && path.accessor->fingerprint) {
+        cacheKey = fetchers::Attrs{
+            {"_what", "fetchToStore"},
+            {"store", store.storeDir},
+            {"name", std::string(name)},
+            {"fingerprint", *path.accessor->fingerprint},
+            {
+                "method",
+                std::visit(overloaded {
+                    [](const TextIngestionMethod &) {
+                        return "text";
+                    },
+                    [](const FileIngestionMethod & fim) {
+                        switch (fim) {
+                        case FileIngestionMethod::Flat: return "flat";
+                        case FileIngestionMethod::Recursive: return "nar";
+                        default: assert(false);
+                        }
+                    },
+                }, method.raw),
+            },
+            {"path", path.path.abs()}
+        };
+        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
+            debug("store path cache hit for '%s'", path);
+            return res->second;
+        }
+    } else
+        debug("source path '%s' is uncacheable", path);
+
+    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path));
+
+    auto filter2 = filter ? *filter : defaultPathFilter;
+
+    auto storePath =
+        settings.readOnlyMode
+        ? store.computeStorePath(
+            name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first
+        : store.addToStore(
+            name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+
+    if (cacheKey)
+        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
+
+    return storePath;
+}
+
+
+}

src/libfetchers/fetch-to-store.hh

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "source-path.hh"
+#include "store-api.hh"
+#include "file-system.hh"
+#include "repair-flag.hh"
+#include "file-content-address.hh"
+
+namespace nix {
+
+/**
+ * Copy the `path` to the Nix store.
+ */
+StorePath fetchToStore(
+    Store & store,
+    const SourcePath & path,
+    std::string_view name = "source",
+    ContentAddressMethod method = FileIngestionMethod::Recursive,
+    PathFilter * filter = nullptr,
+    RepairFlag repair = NoRepair);
+
+}

src/libfetchers/fetchers.cc

@@ -1,6 +1,8 @@
 #include "fetchers.hh"
 #include "store-api.hh"
 #include "input-accessor.hh"
+#include "source-path.hh"
+#include "fetch-to-store.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -374,7 +376,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = SourcePath(accessor).fetchToStore(*store, input2.getName());
+    auto storePath = fetchToStore(*store, SourcePath(accessor), input2.getName());
     return {storePath, input2};
 }
 

src/libfetchers/filtering-input-accessor.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libfetchers/fs-input-accessor.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libfetchers/git.cc

@@ -314,16 +314,27 @@ struct GitInputScheme : InputScheme
 
         writeFile((CanonPath(repoInfo.url) + path).abs(), contents);
 
+        auto result = runProgram(RunOptions {
+            .program = "git",
+            .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
+        });
+        auto exitCode = WEXITSTATUS(result.first);
+
+        if (exitCode != 0) {
+            // The path is not `.gitignore`d, we can add the file.
             runProgram("git", true,
                 { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
+
+            if (commitMsg) {
                 // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
                 logger->pause();
                 Finally restoreLogger([]() { logger->resume(); });
-        if (commitMsg)
                 runProgram("git", true,
                     { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
             }
+        }
+    }
 
     struct RepoInfo
     {

src/libfetchers/input-accessor.cc

@@ -1,129 +0,0 @@
-#include "input-accessor.hh"
-#include "store-api.hh"
-#include "cache.hh"
-
-namespace nix {
-
-StorePath InputAccessor::fetchToStore(
-    Store & store,
-    const CanonPath & path,
-    std::string_view name,
-    ContentAddressMethod method,
-    PathFilter * filter,
-    RepairFlag repair)
-{
-    // FIXME: add an optimisation for the case where the accessor is
-    // an FSInputAccessor pointing to a store path.
-
-    std::optional<fetchers::Attrs> cacheKey;
-
-    if (!filter && fingerprint) {
-        cacheKey = fetchers::Attrs{
-            {"_what", "fetchToStore"},
-            {"store", store.storeDir},
-            {"name", std::string(name)},
-            {"fingerprint", *fingerprint},
-            {
-                "method",
-                std::visit(overloaded {
-                    [](const TextIngestionMethod &) {
-                        return "text";
-                    },
-                    [](const FileIngestionMethod & fim) {
-                        switch (fim) {
-                        case FileIngestionMethod::Flat: return "flat";
-                        case FileIngestionMethod::Recursive: return "nar";
-                        default: assert(false);
-                        }
-                    },
-                }, method.raw),
-            },
-            {"path", path.abs()}
-        };
-        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
-            debug("store path cache hit for '%s'", showPath(path));
-            return res->second;
-        }
-    } else
-        debug("source path '%s' is uncacheable", showPath(path));
-
-    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
-
-    auto filter2 = filter ? *filter : defaultPathFilter;
-
-    auto storePath =
-        settings.readOnlyMode
-        ? store.computeStorePath(
-            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2).first
-        : store.addToStore(
-            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
-
-    if (cacheKey)
-        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
-
-    return storePath;
-}
-
-std::ostream & operator << (std::ostream & str, const SourcePath & path)
-{
-    str << path.to_string();
-    return str;
-}
-
-StorePath SourcePath::fetchToStore(
-    Store & store,
-    std::string_view name,
-    ContentAddressMethod method,
-    PathFilter * filter,
-    RepairFlag repair) const
-{
-    return accessor->fetchToStore(store, path, name, method, filter, repair);
-}
-
-std::string_view SourcePath::baseName() const
-{
-    return path.baseName().value_or("source");
-}
-
-SourcePath SourcePath::parent() const
-{
-    auto p = path.parent();
-    assert(p);
-    return {accessor, std::move(*p)};
-}
-
-SourcePath SourcePath::resolveSymlinks() const
-{
-    auto res = SourcePath(accessor);
-
-    int linksAllowed = 1024;
-
-    std::list<std::string> todo;
-    for (auto & c : path)
-        todo.push_back(std::string(c));
-
-    while (!todo.empty()) {
-        auto c = *todo.begin();
-        todo.pop_front();
-        if (c == "" || c == ".")
-            ;
-        else if (c == "..")
-            res.path.pop();
-        else {
-            res.path.push(c);
-            if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
-                if (!linksAllowed--)
-                    throw Error("infinite symlink recursion in path '%s'", path);
-                auto target = res.readLink();
-                res.path.pop();
-                if (hasPrefix(target, "/"))
-                    res.path = CanonPath::root;
-                todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
-            }
-        }
-    }
-
-    return res;
-}
-
-}

src/libfetchers/input-accessor.hh

@@ -1,174 +0,0 @@
-#pragma once
-///@file
-
-#include "source-accessor.hh"
-#include "ref.hh"
-#include "types.hh"
-#include "file-system.hh"
-#include "repair-flag.hh"
-#include "content-address.hh"
-
-namespace nix {
-
-MakeError(RestrictedPathError, Error);
-
-struct SourcePath;
-class StorePath;
-class Store;
-
-struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
-{
-    std::optional<std::string> fingerprint;
-
-    /**
-     * Return the maximum last-modified time of the files in this
-     * tree, if available.
-     */
-    virtual std::optional<time_t> getLastModified()
-    {
-        return std::nullopt;
-    }
-
-    StorePath fetchToStore(
-        Store & store,
-        const CanonPath & path,
-        std::string_view name = "source",
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
-        RepairFlag repair = NoRepair);
-};
-
-/**
- * An abstraction for accessing source files during
- * evaluation. Currently, it's just a wrapper around `CanonPath` that
- * accesses files in the regular filesystem, but in the future it will
- * support fetching files in other ways.
- */
-struct SourcePath
-{
-    ref<InputAccessor> accessor;
-    CanonPath path;
-
-    SourcePath(ref<InputAccessor> accessor, CanonPath path = CanonPath::root)
-        : accessor(std::move(accessor))
-        , path(std::move(path))
-    { }
-
-    std::string_view baseName() const;
-
-    /**
-     * Construct the parent of this `SourcePath`. Aborts if `this`
-     * denotes the root.
-     */
-    SourcePath parent() const;
-
-    /**
-     * If this `SourcePath` denotes a regular file (not a symlink),
-     * return its contents; otherwise throw an error.
-     */
-    std::string readFile() const
-    { return accessor->readFile(path); }
-
-    /**
-     * Return whether this `SourcePath` denotes a file (of any type)
-     * that exists
-    */
-    bool pathExists() const
-    { return accessor->pathExists(path); }
-
-    /**
-     * Return stats about this `SourcePath`, or throw an exception if
-     * it doesn't exist.
-     */
-    InputAccessor::Stat lstat() const
-    { return accessor->lstat(path); }
-
-    /**
-     * Return stats about this `SourcePath`, or std::nullopt if it
-     * doesn't exist.
-     */
-    std::optional<InputAccessor::Stat> maybeLstat() const
-    { return accessor->maybeLstat(path); }
-
-    /**
-     * If this `SourcePath` denotes a directory (not a symlink),
-     * return its directory entries; otherwise throw an error.
-     */
-    InputAccessor::DirEntries readDirectory() const
-    { return accessor->readDirectory(path); }
-
-    /**
-     * If this `SourcePath` denotes a symlink, return its target;
-     * otherwise throw an error.
-     */
-    std::string readLink() const
-    { return accessor->readLink(path); }
-
-    /**
-     * Dump this `SourcePath` to `sink` as a NAR archive.
-     */
-    void dumpPath(
-        Sink & sink,
-        PathFilter & filter = defaultPathFilter) const
-    { return accessor->dumpPath(path, sink, filter); }
-
-    /**
-     * Copy this `SourcePath` to the Nix store.
-     */
-    StorePath fetchToStore(
-        Store & store,
-        std::string_view name = "source",
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
-        RepairFlag repair = NoRepair) const;
-
-    /**
-     * Return the location of this path in the "real" filesystem, if
-     * it has a physical location.
-     */
-    std::optional<CanonPath> getPhysicalPath() const
-    { return accessor->getPhysicalPath(path); }
-
-    std::string to_string() const
-    { return accessor->showPath(path); }
-
-    /**
-     * Append a `CanonPath` to this path.
-     */
-    SourcePath operator + (const CanonPath & x) const
-    { return {accessor, path + x}; }
-
-    /**
-     * Append a single component `c` to this path. `c` must not
-     * contain a slash. A slash is implicitly added between this path
-     * and `c`.
-     */
-    SourcePath operator + (std::string_view c) const
-    {  return {accessor, path + c}; }
-
-    bool operator == (const SourcePath & x) const
-    {
-        return std::tie(accessor, path) == std::tie(x.accessor, x.path);
-    }
-
-    bool operator != (const SourcePath & x) const
-    {
-        return std::tie(accessor, path) != std::tie(x.accessor, x.path);
-    }
-
-    bool operator < (const SourcePath & x) const
-    {
-        return std::tie(accessor, path) < std::tie(x.accessor, x.path);
-    }
-
-    /**
-     * Resolve any symlinks in this `SourcePath` (including its
-     * parents). The result is a `SourcePath` in which no element is a
-     * symlink.
-     */
-    SourcePath resolveSymlinks() const;
-};
-
-std::ostream & operator << (std::ostream & str, const SourcePath & path);
-
-}

src/libfetchers/local.mk

@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive
+libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive
 
 libfetchers_LIBS = libutil libstore

src/libfetchers/memory-input-accessor.cc

@@ -1,5 +1,6 @@
 #include "memory-input-accessor.hh"
 #include "memory-source-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libfetchers/memory-input-accessor.hh

@@ -1,4 +1,5 @@
 #include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libstore/binary-cache-store.cc

@@ -28,7 +28,8 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
     , Store(params)
 {
     if (secretKeyFile != "")
-        secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
+        signer = std::make_unique<LocalSigner>(
+            SecretKey { readFile(secretKeyFile) });
 
     StringSink sink;
     sink << narVersionMagic1;
@@ -274,7 +275,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
-    if (secretKey) narInfo->sign(*this, *secretKey);
+    if (signer) narInfo->sign(*this, *signer);
 
     writeNarInfo(narInfo);
 

src/libstore/binary-cache-store.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "crypto.hh"
+#include "signature/local-keys.hh"
 #include "store-api.hh"
 #include "log-store.hh"
 
@@ -57,8 +57,7 @@ class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
 {
 
 private:
-
+    std::unique_ptr<Signer> signer;
-    std::unique_ptr<SecretKey> secretKey;
 
 protected:
 

src/libstore/build/substitution-goal.cc

@@ -2,6 +2,7 @@
 #include "substitution-goal.hh"
 #include "nar-info.hh"
 #include "finally.hh"
+#include "signals.hh"
 
 namespace nix {
 
@@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun()
 
     thr = std::thread([this]() {
         try {
+            ReceiveInterrupts receiveInterrupts;
+
             /* Wake up the worker loop when we're done. */
             Finally updateStats([this]() { outPipe.writeSide.close(); });
 

src/libstore/derivations.cc

@@ -2,6 +2,7 @@
 #include "downstream-placeholder.hh"
 #include "store-api.hh"
 #include "globals.hh"
+#include "types.hh"
 #include "util.hh"
 #include "split.hh"
 #include "common-protocol.hh"
@@ -154,31 +155,78 @@ StorePath writeDerivation(Store & store,
 }
 
 
-/* Read string `s' from stream `str'. */
+namespace {
-static void expect(std::istream & str, std::string_view s)
+/**
-{
+ * This mimics std::istream to some extent. We use this much smaller implementation
-    for (auto & c : s) {
+ * instead of plain istreams because the sentry object overhead is too high.
-        if (str.get() != c)
+ */
-            throw FormatError("expected string '%1%'", s);
+struct StringViewStream {
+    std::string_view remaining;
+
+    int peek() const {
+        return remaining.empty() ? EOF : remaining[0];
     }
+
+    int get() {
+        if (remaining.empty()) return EOF;
+        char c = remaining[0];
+        remaining.remove_prefix(1);
+        return c;
+    }
+};
+
+constexpr struct Escapes {
+    char map[256];
+    constexpr Escapes() {
+        for (int i = 0; i < 256; i++) map[i] = (char) (unsigned char) i;
+        map[(int) (unsigned char) 'n'] = '\n';
+        map[(int) (unsigned char) 'r'] = '\r';
+        map[(int) (unsigned char) 't'] = '\t';
+    }
+    char operator[](char c) const { return map[(unsigned char) c]; }
+} escapes;
+}
+
+
+/* Read string `s' from stream `str'. */
+static void expect(StringViewStream & str, std::string_view s)
+{
+    if (!str.remaining.starts_with(s))
+        throw FormatError("expected string '%1%'", s);
+    str.remaining.remove_prefix(s.size());
 }
 
 
 /* Read a C-style string from stream `str'. */
-static std::string parseString(std::istream & str)
+static BackedStringView parseString(StringViewStream & str)
 {
-    std::string res;
     expect(str, "\"");
-    int c;
+    auto c = str.remaining.begin(), end = str.remaining.end();
-    while ((c = str.get()) != '"')
+    bool escaped = false;
-        if (c == '\\') {
+    for (; c != end && *c != '"'; c++) {
-            c = str.get();
+        if (*c == '\\') {
-            if (c == 'n') res += '\n';
+            c++;
-            else if (c == 'r') res += '\r';
+            if (c == end)
-            else if (c == 't') res += '\t';
+                throw FormatError("unterminated string in derivation");
-            else res += c;
+            escaped = true;
         }
-        else res += c;
+    }
+
+    const auto contentLen = c - str.remaining.begin();
+    const auto content = str.remaining.substr(0, contentLen);
+    str.remaining.remove_prefix(contentLen + 1);
+
+    if (!escaped)
+        return content;
+
+    std::string res;
+    res.reserve(content.size());
+    for (c = content.begin(), end = content.end(); c != end; c++)
+        if (*c == '\\') {
+            c++;
+            res += escapes[*c];
+        }
+        else res += *c;
     return res;
 }
 
@@ -187,15 +235,15 @@ static void validatePath(std::string_view s) {
         throw FormatError("bad path '%1%' in derivation", s);
 }
 
-static Path parsePath(std::istream & str)
+static BackedStringView parsePath(StringViewStream & str)
 {
     auto s = parseString(str);
-    validatePath(s);
+    validatePath(*s);
     return s;
 }
 
 
-static bool endOfList(std::istream & str)
+static bool endOfList(StringViewStream & str)
 {
     if (str.peek() == ',') {
         str.get();
@@ -209,12 +257,12 @@ static bool endOfList(std::istream & str)
 }
 
 
-static StringSet parseStrings(std::istream & str, bool arePaths)
+static StringSet parseStrings(StringViewStream & str, bool arePaths)
 {
     StringSet res;
     expect(str, "[");
     while (!endOfList(str))
-        res.insert(arePaths ? parsePath(str) : parseString(str));
+        res.insert((arePaths ? parsePath(str) : parseString(str)).toOwned());
     return res;
 }
 
@@ -267,7 +315,7 @@ static DerivationOutput parseDerivationOutput(
 }
 
 static DerivationOutput parseDerivationOutput(
-    const StoreDirConfig & store, std::istringstream & str,
+    const StoreDirConfig & store, StringViewStream & str,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
@@ -275,7 +323,7 @@ static DerivationOutput parseDerivationOutput(
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");
 
-    return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings);
+    return parseDerivationOutput(store, *pathS, *hashAlgo, *hash, xpSettings);
 }
 
 /**
@@ -297,7 +345,7 @@ enum struct DerivationATermVersion {
 
 static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
     const StoreDirConfig & store,
-    std::istringstream & str,
+    StringViewStream & str,
     DerivationATermVersion version)
 {
     DerivedPathMap<StringSet>::ChildNode node;
@@ -323,7 +371,7 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
             expect(str, ",[");
             while (!endOfList(str)) {
                 expect(str, "(");
-                auto outputName = parseString(str);
+                auto outputName = parseString(str).toOwned();
                 expect(str, ",");
                 node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version));
                 expect(str, ")");
@@ -349,7 +397,7 @@ Derivation parseDerivation(
     Derivation drv;
     drv.name = name;
 
-    std::istringstream str(std::move(s));
+    StringViewStream str{s};
     expect(str, "D");
     DerivationATermVersion version;
     switch (str.peek()) {
@@ -360,12 +408,12 @@ Derivation parseDerivation(
     case 'r': {
         expect(str, "rvWithVersion(");
         auto versionS = parseString(str);
-        if (versionS == "xp-dyn-drv") {
+        if (*versionS == "xp-dyn-drv") {
             // Only verison we have so far
             version = DerivationATermVersion::DynamicDerivations;
             xpSettings.require(Xp::DynamicDerivations);
         } else {
-            throw FormatError("Unknown derivation ATerm format version '%s'", versionS);
+            throw FormatError("Unknown derivation ATerm format version '%s'", *versionS);
         }
         expect(str, ",");
         break;
@@ -377,7 +425,7 @@ Derivation parseDerivation(
     /* Parse the list of outputs. */
     expect(str, "[");
     while (!endOfList(str)) {
-        expect(str, "("); std::string id = parseString(str);
+        expect(str, "("); std::string id = parseString(str).toOwned();
         auto output = parseDerivationOutput(store, str, xpSettings);
         drv.outputs.emplace(std::move(id), std::move(output));
     }
@@ -386,28 +434,28 @@ Derivation parseDerivation(
     expect(str, ",[");
     while (!endOfList(str)) {
         expect(str, "(");
-        Path drvPath = parsePath(str);
+        auto drvPath = parsePath(str);
         expect(str, ",");
-        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version));
+        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version));
         expect(str, ")");
     }
 
     expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
-    expect(str, ","); drv.platform = parseString(str);
+    expect(str, ","); drv.platform = parseString(str).toOwned();
-    expect(str, ","); drv.builder = parseString(str);
+    expect(str, ","); drv.builder = parseString(str).toOwned();
 
     /* Parse the builder arguments. */
     expect(str, ",[");
     while (!endOfList(str))
-        drv.args.push_back(parseString(str));
+        drv.args.push_back(parseString(str).toOwned());
 
     /* Parse the environment variables. */
     expect(str, ",[");
     while (!endOfList(str)) {
-        expect(str, "("); auto name = parseString(str);
+        expect(str, "("); auto name = parseString(str).toOwned();
-        expect(str, ","); auto value = parseString(str);
+        expect(str, ","); auto value = parseString(str).toOwned();
         expect(str, ")");
-        drv.env[name] = value;
+        drv.env.insert_or_assign(std::move(name), std::move(value));
     }
 
     expect(str, ")");

src/libstore/derived-path.cc

@@ -12,9 +12,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@@ -22,13 +22,9 @@ namespace nix {
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
 
-#define FIELD_TYPE std::string
 CMP(SingleDerivedPath, SingleDerivedPathBuilt, output)
-#undef FIELD_TYPE
 
-#define FIELD_TYPE OutputsSpec
 CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
-#undef FIELD_TYPE
 
 #undef CMP
 #undef CMP_ONE

src/libstore/globals.cc

@@ -15,8 +15,6 @@
 
 #include <nlohmann/json.hpp>
 
-#include <sodium/core.h>
-
 #ifdef __GLIBC__
 # include <gnu/lib-names.h>
 # include <nss.h>
@@ -409,9 +407,6 @@ void initLibStore() {
 
     initLibUtil();
 
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
-
     loadConfFile();
 
     preloadNSS();

src/libstore/globals.hh

@@ -144,7 +144,7 @@ public:
      */
     bool verboseBuild = true;
 
-    Setting<size_t> logLines{this, 10, "log-lines",
+    Setting<size_t> logLines{this, 25, "log-lines",
         "The number of lines of the tail of "
         "the log to show if a build fails."};
 
@@ -946,7 +946,9 @@ public:
           may be useful in certain scenarios (e.g. to spin up containers or
           set up userspace network interfaces in tests).
         )"};
+#endif
 
+#if HAVE_ACL_SUPPORT
     Setting<StringSet> ignoredAcls{
         this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls",
         R"(

src/libstore/keys.cc

@@ -0,0 +1,31 @@
+#include "file-system.hh"
+#include "globals.hh"
+#include "keys.hh"
+
+namespace nix {
+
+PublicKeys getDefaultPublicKeys()
+{
+    PublicKeys publicKeys;
+
+    // FIXME: filter duplicates
+
+    for (auto s : settings.trustedPublicKeys.get()) {
+        PublicKey key(s);
+        publicKeys.emplace(key.name, key);
+    }
+
+    for (auto secretKeyFile : settings.secretKeyFiles.get()) {
+        try {
+            SecretKey secretKey(readFile(secretKeyFile));
+            publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
+        } catch (SysError & e) {
+            /* Ignore unreadable key files. That's normal in a
+               multi-user installation. */
+        }
+    }
+
+    return publicKeys;
+}
+
+}

src/libstore/keys.hh

@@ -0,0 +1,10 @@
+#pragma once
+///@file
+
+#include "signature/local-keys.hh"
+
+namespace nix {
+
+PublicKeys getDefaultPublicKeys();
+
+}

src/libstore/local-store.cc

@@ -14,11 +14,14 @@
 #include "signals.hh"
 #include "posix-fs-canonicalise.hh"
 #include "posix-source-accessor.hh"
+#include "keys.hh"
 
 #include <iostream>
 #include <algorithm>
 #include <cstring>
 
+#include <memory>
+#include <new>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/select.h>
@@ -1147,7 +1150,11 @@ StorePath LocalStore::addToStoreFromDump(
        path. */
     bool inMemory = false;
 
-    std::string dump;
+    struct Free {
+        void operator()(void* v) { free(v); }
+    };
+    std::unique_ptr<char, Free> dumpBuffer(nullptr);
+    std::string_view dump;
 
     /* Fill out buffer, and decide whether we are working strictly in
        memory based on whether we break out because the buffer is full
@@ -1156,13 +1163,18 @@ StorePath LocalStore::addToStoreFromDump(
         auto oldSize = dump.size();
         constexpr size_t chunkSize = 65536;
         auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
-        dump.resize(oldSize + want);
+        if (auto tmp = realloc(dumpBuffer.get(), oldSize + want)) {
+            dumpBuffer.release();
+            dumpBuffer.reset((char*) tmp);
+        } else {
+            throw std::bad_alloc();
+        }
         auto got = 0;
         Finally cleanup([&]() {
-            dump.resize(oldSize + got);
+            dump = {dumpBuffer.get(), dump.size() + got};
         });
         try {
-            got = source.read(dump.data() + oldSize, want);
+            got = source.read(dumpBuffer.get() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
@@ -1185,7 +1197,8 @@ StorePath LocalStore::addToStoreFromDump(
 
         restorePath(tempPath, bothSource, method.getFileIngestionMethod());
 
-        dump.clear();
+        dumpBuffer.reset();
+        dump = {};
     }
 
     auto [hash, size] = hashSink->finish();
@@ -1566,7 +1579,8 @@ void LocalStore::signRealisation(Realisation & realisation)
 
     for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
-        realisation.sign(secretKey);
+        LocalSigner signer(std::move(secretKey));
+        realisation.sign(signer);
     }
 }
 
@@ -1578,7 +1592,8 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 
     for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
-        info.sign(*this, secretKey);
+        LocalSigner signer(std::move(secretKey));
+        info.sign(*this, signer);
     }
 }
 

src/libstore/local.mk

@@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
 libstore_LIBS = libutil
 
-libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
  libstore_LDFLAGS += -ldl
 endif
@@ -48,9 +48,9 @@ $(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell)
 	$(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp
 	@mv $@.tmp $@
 else
-ifneq ($(sandbox_shell),)
+  ifneq ($(sandbox_shell),)
-libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
+    libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
-endif
+  endif
 endif
 
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh

src/libstore/path-info.cc

@@ -38,9 +38,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
 }
 
 
-void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
+void ValidPathInfo::sign(const Store & store, const Signer & signer)
 {
-    sigs.insert(secretKey.signDetached(fingerprint(store)));
+    sigs.insert(signer.signDetached(fingerprint(store)));
 }
 
 std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const

src/libstore/path-info.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "crypto.hh"
+#include "signature/signer.hh"
 #include "path.hh"
 #include "hash.hh"
 #include "content-address.hh"
@@ -107,7 +107,7 @@ struct ValidPathInfo : UnkeyedValidPathInfo {
      */
     std::string fingerprint(const Store & store) const;
 
-    void sign(const Store & store, const SecretKey & secretKey);
+    void sign(const Store & store, const Signer & signer);
 
     /**
      * @return The `ContentAddressWithReferences` that determines the

src/libstore/path.cc

@@ -1,7 +1,5 @@
 #include "store-dir-config.hh"
 
-#include <sodium.h>
-
 namespace nix {
 
 static void checkName(std::string_view path, std::string_view name)
@@ -49,9 +47,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath StorePath::random(std::string_view name)
 {
-    Hash hash(HashAlgorithm::SHA1);
+    return StorePath(Hash::random(HashAlgorithm::SHA1), name);
-    randombytes_buf(hash.hash, hash.hashSize);
-    return StorePath(hash, name);
 }
 
 StorePath StoreDirConfig::parseStorePath(std::string_view path) const

src/libstore/posix-fs-canonicalise.cc

@@ -1,4 +1,4 @@
-#if HAVE_SYS_XATTR_H
+#if HAVE_ACL_SUPPORT
 # include <sys/xattr.h>
 #endif
 
@@ -78,7 +78,7 @@ static void canonicalisePathMetaData_(
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error("file '%1%' has an unsupported type", path);
 
-#if HAVE_SYS_XATTR_H && HAVE_LLISTXATTR && HAVE_LREMOVEXATTR
+#if HAVE_ACL_SUPPORT
     /* Remove extended attributes / ACLs. */
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 

src/libstore/realisation.cc

@@ -1,6 +1,7 @@
 #include "realisation.hh"
 #include "store-api.hh"
 #include "closure.hh"
+#include "signature/local-keys.hh"
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -113,9 +114,9 @@ std::string Realisation::fingerprint() const
     return serialized.dump();
 }
 
-void Realisation::sign(const SecretKey & secretKey)
+void Realisation::sign(const Signer &signer)
 {
-    signatures.insert(secretKey.signDetached(fingerprint()));
+    signatures.insert(signer.signDetached(fingerprint()));
 }
 
 bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const

src/libstore/realisation.hh

@@ -8,7 +8,7 @@
 #include "derived-path.hh"
 #include <nlohmann/json_fwd.hpp>
 #include "comparator.hh"
-#include "crypto.hh"
+#include "signature/signer.hh"
 
 namespace nix {
 
@@ -64,7 +64,7 @@ struct Realisation {
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 
     std::string fingerprint() const;
-    void sign(const SecretKey &);
+    void sign(const Signer &);
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 

src/libstore/remote-store.cc

@@ -16,6 +16,8 @@
 #include "logging.hh"
 #include "callback.hh"
 #include "filetransfer.hh"
+#include "signals.hh"
+
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -186,7 +188,7 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source,
                 if (m.find("parsing derivation") != std::string::npos &&
                     m.find("expected string") != std::string::npos &&
                     m.find("Derive([") != std::string::npos)
-                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
+                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
             }
             throw;
         }
@@ -1066,6 +1068,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sin
     std::thread stderrThread([&]()
     {
         try {
+            ReceiveInterrupts receiveInterrupts;
             processStderr(nullptr, nullptr, false);
         } catch (...) {
             ex = std::current_exception();

src/libstore/store-api.cc

@@ -1,4 +1,4 @@
-#include "crypto.hh"
+#include "signature/local-keys.hh"
 #include "source-accessor.hh"
 #include "globals.hh"
 #include "derived-path.hh"
@@ -194,7 +194,10 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
     if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
     } else {
-        assert(info.references.size() == 0);
+        if (!info.references.empty()) {
+            throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.",
+                name);
+        }
         return makeStorePath("output:out",
             hashString(HashAlgorithm::SHA256,
                 "fixed:out:"

src/libstore/store-api.hh

@@ -13,6 +13,7 @@
 #include "path-info.hh"
 #include "repair-flag.hh"
 #include "store-dir-config.hh"
+#include "source-path.hh"
 
 #include <nlohmann/json_fwd.hpp>
 #include <atomic>

src/libutil/archive.cc

@@ -140,7 +140,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
     sink.preallocateContents(size);
 
     uint64_t left = size;
-    std::vector<char> buf(65536);
+    std::array<char, 65536> buf;
 
     while (left) {
         checkInterrupt();

src/libutil/comparator.hh

@@ -13,9 +13,9 @@
 #define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \
     PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \
       __VA_OPT__(const MY_TYPE * me = this;) \
-      auto fields1 = std::make_tuple( __VA_ARGS__ ); \
+      auto fields1 = std::tie( __VA_ARGS__ ); \
       __VA_OPT__(me = &other;) \
-      auto fields2 = std::make_tuple( __VA_ARGS__ ); \
+      auto fields2 = std::tie( __VA_ARGS__ ); \
       return fields1 COMPARATOR fields2; \
     }
 #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \

src/libutil/english.cc

@@ -0,0 +1,18 @@
+#include "english.hh"
+
+namespace nix {
+
+std::ostream & pluralize(
+    std::ostream & output,
+    unsigned int count,
+    const std::string_view single,
+    const std::string_view plural)
+{
+    if (count == 1)
+        output << "1 " << single;
+    else
+        output << count << " " << plural;
+    return output;
+}
+
+}

src/libutil/english.hh

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <iostream>
+
+namespace nix {
+
+/**
+ * Pluralize a given value.
+ *
+ * If `count == 1`, prints `1 {single}` to `output`, otherwise prints `{count} {plural}`.
+ */
+std::ostream & pluralize(
+    std::ostream & output,
+    unsigned int count,
+    const std::string_view single,
+    const std::string_view plural);
+
+}

src/libutil/error.cc

@@ -2,6 +2,7 @@
 #include "environment-variables.hh"
 #include "signals.hh"
 #include "terminal.hh"
+#include "position.hh"
 
 #include <iostream>
 #include <optional>
@@ -10,7 +11,7 @@
 
 namespace nix {
 
-void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame)
+void BaseError::addTrace(std::shared_ptr<Pos> && e, hintformat hint, bool frame)
 {
     err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
 }
@@ -41,58 +42,36 @@ std::ostream & operator <<(std::ostream & os, const hintformat & hf)
     return os << hf.str();
 }
 
-std::ostream & operator <<(std::ostream & str, const AbstractPos & pos)
+/**
+ * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container.
+ */
+inline bool operator<(const Trace& lhs, const Trace& rhs)
 {
-    pos.print(str);
+    // `std::shared_ptr` does not have value semantics for its comparison
-    str << ":" << pos.line;
+    // functions, so we need to check for nulls and compare the dereferenced
-    if (pos.column > 0)
+    // values here.
-        str << ":" << pos.column;
+    if (lhs.pos != rhs.pos) {
-    return str;
+        if (!lhs.pos)
-}
+            return true;
-
+        if (!rhs.pos)
-std::optional<LinesOfCode> AbstractPos::getCodeLines() const
+            return false;
-{
+        if (*lhs.pos != *rhs.pos)
-    if (line == 0)
+            return *lhs.pos < *rhs.pos;
-        return std::nullopt;
-
-    if (auto source = getSource()) {
-
-        std::istringstream iss(*source);
-        // count the newlines.
-        int count = 0;
-        std::string curLine;
-        int pl = line - 1;
-
-        LinesOfCode loc;
-
-        do {
-            std::getline(iss, curLine);
-            ++count;
-            if (count < pl)
-                ;
-            else if (count == pl) {
-                loc.prevLineOfCode = curLine;
-            } else if (count == pl + 1) {
-                loc.errLineOfCode = curLine;
-            } else if (count == pl + 2) {
-                loc.nextLineOfCode = curLine;
-                break;
     }
-
+    // This formats a freshly formatted hint string and then throws it away, which
-            if (!iss.good())
+    // shouldn't be much of a problem because it only runs when pos is equal, and this function is
-                break;
+    // used for trace printing, which is infrequent.
-        } while (true);
+    return std::forward_as_tuple(lhs.hint.str(), lhs.frame)
-
+        < std::forward_as_tuple(rhs.hint.str(), rhs.frame);
-        return loc;
-    }
-
-    return std::nullopt;
 }
+inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; }
+inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); }
+inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); }
 
 // print lines of code to the ostream, indicating the error column.
 void printCodeLines(std::ostream & out,
     const std::string & prefix,
-    const AbstractPos & errPos,
+    const Pos & errPos,
     const LinesOfCode & loc)
 {
     // previous line of code.
@@ -170,7 +149,7 @@ static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").h
  *
  * @return true if a position was printed.
  */
-static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr<AbstractPos> & pos) {
+static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr<Pos> & pos) {
     bool hasPos = pos && *pos;
     if (hasPos) {
         oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":";
@@ -185,6 +164,69 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std
     return hasPos;
 }
 
+void printTrace(
+    std::ostream & output,
+    const std::string_view & indent,
+    size_t & count,
+    const Trace & trace)
+{
+    output << "\n" << "… " << trace.hint.str() << "\n";
+
+    if (printPosMaybe(output, indent, trace.pos))
+        count++;
+}
+
+void printSkippedTracesMaybe(
+    std::ostream & output,
+    const std::string_view & indent,
+    size_t & count,
+    std::vector<Trace> & skippedTraces,
+    std::set<Trace> tracesSeen)
+{
+    if (skippedTraces.size() > 0) {
+        // If we only skipped a few frames, print them out normally;
+        // messages like "1 duplicate frames omitted" aren't helpful.
+        if (skippedTraces.size() <= 5) {
+            for (auto & trace : skippedTraces) {
+                printTrace(output, indent, count, trace);
+            }
+        } else {
+            output << "\n" << ANSI_WARNING "(" << skippedTraces.size() << " duplicate frames omitted)" ANSI_NORMAL << "\n";
+            // Clear the set of "seen" traces after printing a chunk of
+            // `duplicate frames omitted`.
+            //
+            // Consider a mutually recursive stack trace with:
+            // - 10 entries of A
+            // - 10 entries of B
+            // - 10 entries of A
+            //
+            // If we don't clear `tracesSeen` here, we would print output like this:
+            // - 1 entry of A
+            // - (9 duplicate frames omitted)
+            // - 1 entry of B
+            // - (19 duplicate frames omitted)
+            //
+            // This would obscure the control flow, which went from A,
+            // to B, and back to A again.
+            //
+            // In contrast, if we do clear `tracesSeen`, the output looks like this:
+            // - 1 entry of A
+            // - (9 duplicate frames omitted)
+            // - 1 entry of B
+            // - (9 duplicate frames omitted)
+            // - 1 entry of A
+            // - (9 duplicate frames omitted)
+            //
+            // See: `tests/functional/lang/eval-fail-mutual-recursion.nix`
+            tracesSeen.clear();
+        }
+    }
+    // We've either printed each trace in `skippedTraces` normally, or
+    // printed a chunk of `duplicate frames omitted`. Either way, we've
+    // processed these traces and can clear them.
+    skippedTraces.clear();
+}
+
 std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
 {
     std::string prefix;
@@ -333,7 +375,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     bool frameOnly = false;
     if (!einfo.traces.empty()) {
+        // Stack traces seen since we last printed a chunk of `duplicate frames
+        // omitted`.
+        std::set<Trace> tracesSeen;
+        // A consecutive sequence of stack traces that are all in `tracesSeen`.
+        std::vector<Trace> skippedTraces;
         size_t count = 0;
+
         for (const auto & trace : einfo.traces) {
             if (trace.hint.str().empty()) continue;
             if (frameOnly && !trace.frame) continue;
@@ -343,14 +391,21 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
                 break;
             }
 
+            if (tracesSeen.count(trace)) {
+                skippedTraces.push_back(trace);
+                continue;
+            }
+            tracesSeen.insert(trace);
+
+            printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
+
             count++;
             frameOnly = trace.frame;
 
-            oss << "\n" << "… " << trace.hint.str() << "\n";
+            printTrace(oss, ellipsisIndent, count, trace);
-
-            if (printPosMaybe(oss, ellipsisIndent, trace.pos))
-                count++;
         }
+
+        printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
         oss << "\n" << prefix;
     }
 
@@ -369,4 +424,5 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     return out;
 }
+
 }

src/libutil/error.hh

@@ -25,6 +25,7 @@
 #include <memory>
 #include <map>
 #include <optional>
+#include <compare>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -62,51 +63,28 @@ struct LinesOfCode {
     std::optional<std::string> nextLineOfCode;
 };
 
-/**
+struct Pos;
- * An abstract type that represents a location in a source file.
- */
-struct AbstractPos
-{
-    uint32_t line = 0;
-    uint32_t column = 0;
-
-    /**
-     * An AbstractPos may be a "null object", representing an unknown position.
-     *
-     * Return true if this position is known.
-     */
-    inline operator bool() const { return line != 0; };
-
-    /**
-     * Return the contents of the source file.
-     */
-    virtual std::optional<std::string> getSource() const
-    { return std::nullopt; };
-
-    virtual void print(std::ostream & out) const = 0;
-
-    std::optional<LinesOfCode> getCodeLines() const;
-
-    virtual ~AbstractPos() = default;
-};
-
-std::ostream & operator << (std::ostream & str, const AbstractPos & pos);
 
 void printCodeLines(std::ostream & out,
     const std::string & prefix,
-    const AbstractPos & errPos,
+    const Pos & errPos,
     const LinesOfCode & loc);
 
 struct Trace {
-    std::shared_ptr<AbstractPos> pos;
+    std::shared_ptr<Pos> pos;
     hintformat hint;
     bool frame;
 };
 
+inline bool operator<(const Trace& lhs, const Trace& rhs);
+inline bool operator> (const Trace& lhs, const Trace& rhs);
+inline bool operator<=(const Trace& lhs, const Trace& rhs);
+inline bool operator>=(const Trace& lhs, const Trace& rhs);
+
 struct ErrorInfo {
     Verbosity level;
     hintformat msg;
-    std::shared_ptr<AbstractPos> errPos;
+    std::shared_ptr<Pos> errPos;
     std::list<Trace> traces;
 
     Suggestions suggestions;
@@ -177,12 +155,12 @@ public:
     }
 
     template<typename... Args>
-    void addTrace(std::shared_ptr<AbstractPos> && e, std::string_view fs, const Args & ... args)
+    void addTrace(std::shared_ptr<Pos> && e, std::string_view fs, const Args & ... args)
     {
         addTrace(std::move(e), hintfmt(std::string(fs), args...));
     }
 
-    void addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame = false);
+    void addTrace(std::shared_ptr<Pos> && e, hintformat hint, bool frame = false);
 
     bool hasTrace() const { return !err.traces.empty(); }
 

src/libutil/file-system.cc

@@ -307,7 +307,7 @@ void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
     if (!fd)
         throw SysError("opening file '%1%'", path);
 
-    std::vector<char> buf(64 * 1024);
+    std::array<char, 64 * 1024> buf;
 
     try {
         while (true) {

src/libutil/hash.cc

@@ -14,6 +14,8 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
+#include <sodium.h>
+
 namespace nix {
 
 static size_t regularHashSize(HashAlgorithm type) {
@@ -261,6 +263,13 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
         throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo));
 }
 
+Hash Hash::random(HashAlgorithm algo)
+{
+    Hash hash(algo);
+    randombytes_buf(hash.hash, hash.hashSize);
+    return hash;
+}
+
 Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha)
 {
     if (hashStr.empty()) {

src/libutil/hash.hh

@@ -5,7 +5,6 @@
 #include "serialise.hh"
 #include "file-system.hh"
 
-
 namespace nix {
 
 
@@ -143,6 +142,11 @@ public:
     }
 
     static Hash dummy;
+
+    /**
+     * @return a random hash with hash algorithm `algo`
+     */
+    static Hash random(HashAlgorithm algo);
 };
 
 /**

src/libutil/input-accessor.hh

@@ -0,0 +1,27 @@
+#pragma once
+///@file
+
+#include "source-accessor.hh"
+#include "ref.hh"
+#include "repair-flag.hh"
+
+namespace nix {
+
+MakeError(RestrictedPathError, Error);
+
+struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
+{
+    std::optional<std::string> fingerprint;
+
+    /**
+     * Return the maximum last-modified time of the files in this
+     * tree, if available.
+     */
+    virtual std::optional<time_t> getLastModified()
+    {
+        return std::nullopt;
+    }
+
+};
+
+}

src/libutil/local.mk

@@ -4,14 +4,17 @@ libutil_NAME = libnixutil
 
 libutil_DIR := $(d)
 
-libutil_SOURCES := $(wildcard $(d)/*.cc)
+libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc)
 
 libutil_CXXFLAGS += -I src/libutil
 
-libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 $(foreach i, $(wildcard $(d)/args/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644)))
+$(foreach i, $(wildcard $(d)/signature/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/signature, 0644)))
+
 
 ifeq ($(HAVE_LIBCPUID), 1)
   libutil_LDFLAGS += -lcpuid

src/libutil/logging.cc

@@ -4,6 +4,8 @@
 #include "terminal.hh"
 #include "util.hh"
 #include "config.hh"
+#include "source-path.hh"
+#include "position.hh"
 
 #include <atomic>
 #include <nlohmann/json.hpp>
@@ -136,13 +138,13 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
     logger.startActivity(id, lvl, type, s, fields, parent);
 }
 
-void to_json(nlohmann::json & json, std::shared_ptr<AbstractPos> pos)
+void to_json(nlohmann::json & json, std::shared_ptr<Pos> pos)
 {
     if (pos) {
         json["line"] = pos->line;
         json["column"] = pos->column;
         std::ostringstream str;
-        pos->print(str);
+        pos->print(str, true);
         json["file"] = str.str();
     } else {
         json["line"] = nullptr;

src/libutil/position.cc

@@ -0,0 +1,112 @@
+#include "position.hh"
+
+namespace nix {
+
+Pos::Pos(const Pos * other)
+{
+    if (!other) {
+        return;
+    }
+    line = other->line;
+    column = other->column;
+    origin = std::move(other->origin);
+}
+
+Pos::operator std::shared_ptr<Pos>() const
+{
+    return std::make_shared<Pos>(&*this);
+}
+
+bool Pos::operator<(const Pos &rhs) const
+{
+    return std::forward_as_tuple(line, column, origin)
+        < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin);
+}
+
+std::optional<LinesOfCode> Pos::getCodeLines() const
+{
+    if (line == 0)
+        return std::nullopt;
+
+    if (auto source = getSource()) {
+
+        std::istringstream iss(*source);
+        // count the newlines.
+        int count = 0;
+        std::string curLine;
+        int pl = line - 1;
+
+        LinesOfCode loc;
+
+        do {
+            std::getline(iss, curLine);
+            ++count;
+            if (count < pl)
+                ;
+            else if (count == pl) {
+                loc.prevLineOfCode = curLine;
+            } else if (count == pl + 1) {
+                loc.errLineOfCode = curLine;
+            } else if (count == pl + 2) {
+                loc.nextLineOfCode = curLine;
+                break;
+            }
+
+            if (!iss.good())
+                break;
+        } while (true);
+
+        return loc;
+    }
+
+    return std::nullopt;
+}
+
+
+std::optional<std::string> Pos::getSource() const
+{
+    return std::visit(overloaded {
+        [](const std::monostate &) -> std::optional<std::string> {
+            return std::nullopt;
+        },
+        [](const Pos::Stdin & s) -> std::optional<std::string> {
+            // Get rid of the null terminators added by the parser.
+            return std::string(s.source->c_str());
+        },
+        [](const Pos::String & s) -> std::optional<std::string> {
+            // Get rid of the null terminators added by the parser.
+            return std::string(s.source->c_str());
+        },
+        [](const SourcePath & path) -> std::optional<std::string> {
+            try {
+                return path.readFile();
+            } catch (Error &) {
+                return std::nullopt;
+            }
+        }
+    }, origin);
+}
+
+void Pos::print(std::ostream & out, bool showOrigin) const
+{
+    if (showOrigin) {
+        std::visit(overloaded {
+            [&](const std::monostate &) { out << "«none»"; },
+            [&](const Pos::Stdin &) { out << "«stdin»"; },
+            [&](const Pos::String & s) { out << "«string»"; },
+            [&](const SourcePath & path) { out << path; }
+        }, origin);
+        out << ":";
+    }
+    out << line;
+    if (column > 0)
+        out << ":" << column;
+}
+
+std::ostream & operator<<(std::ostream & str, const Pos & pos)
+{
+    pos.print(str, true);
+    return str;
+}
+
+}

src/libutil/position.hh

@@ -0,0 +1,74 @@
+#pragma once
+/**
+ * @file
+ *
+ * @brief Pos and AbstractPos
+ */
+
+#include <cstdint>
+#include <string>
+
+#include "source-path.hh"
+
+namespace nix {
+
+/**
+ * A position and an origin for that position (like a source file).
+ */
+struct Pos
+{
+    uint32_t line = 0;
+    uint32_t column = 0;
+
+    struct Stdin {
+        ref<std::string> source;
+        bool operator==(const Stdin & rhs) const
+        { return *source == *rhs.source; }
+        bool operator!=(const Stdin & rhs) const
+        { return *source != *rhs.source; }
+        bool operator<(const Stdin & rhs) const
+        { return *source < *rhs.source; }
+    };
+    struct String {
+        ref<std::string> source;
+        bool operator==(const String & rhs) const
+        { return *source == *rhs.source; }
+        bool operator!=(const String & rhs) const
+        { return *source != *rhs.source; }
+        bool operator<(const String & rhs) const
+        { return *source < *rhs.source; }
+    };
+
+    typedef std::variant<std::monostate, Stdin, String, SourcePath> Origin;
+
+    Origin origin = std::monostate();
+
+    Pos() { }
+    Pos(uint32_t line, uint32_t column, Origin origin)
+        : line(line), column(column), origin(origin) { }
+    Pos(Pos & other) = default;
+    Pos(const Pos & other) = default;
+    Pos(Pos && other) = default;
+    Pos(const Pos * other);
+
+    explicit operator bool() const { return line > 0; }
+
+    operator std::shared_ptr<Pos>() const;
+
+    /**
+     * Return the contents of the source file.
+     */
+    std::optional<std::string> getSource() const;
+
+    void print(std::ostream & out, bool showOrigin) const;
+
+    std::optional<LinesOfCode> getCodeLines() const;
+
+    bool operator==(const Pos & rhs) const = default;
+    bool operator!=(const Pos & rhs) const = default;
+    bool operator<(const Pos & rhs) const;
+};
+
+std::ostream & operator<<(std::ostream & str, const Pos & pos);
+
+}

src/libutil/posix-source-accessor.cc

@@ -25,7 +25,7 @@ void PosixSourceAccessor::readFile(
 
     off_t left = st.st_size;
 
-    std::vector<unsigned char> buf(64 * 1024);
+    std::array<unsigned char, 64 * 1024> buf;
     while (left) {
         checkInterrupt();
         ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));

src/libutil/ref.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include <compare>
 #include <memory>
 #include <exception>
 #include <stdexcept>