Merge remote-tracking branch 'origin/master' into flake-regressions

.editorconfig

@@ -4,20 +4,20 @@
 # Top-most EditorConfig file
 root = true
 
-# Unix-style newlines with a newline ending every file, utf-8 charset
+# Unix-style newlines with a newline ending every file, UTF-8 charset
 [*]
 end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 charset = utf-8
 
-# Match nix files, set indent to spaces with width of two
+# Match Nix files, set indent to spaces with width of two
 [*.nix]
 indent_style = space
 indent_size = 2
 
-# Match c++/shell/perl, set indent to spaces with width of four
+# Match C++/C/shell/Perl, set indent to spaces with width of four
-[*.{hpp,cc,hh,sh,pl,xs}]
+[*.{hpp,cc,hh,c,h,sh,pl,xs}]
 indent_style = space
 indent_size = 4
 

.github/CODEOWNERS

@@ -23,4 +23,4 @@ maintainers/*.md @fricklerhandwerk
 src/**/*.md @fricklerhandwerk
 
 # Libstore layer
-/src/libstore @thufschmitt @ericson2314
+/src/libstore @ericson2314

.github/workflows/ci.yml

@@ -31,6 +31,23 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - if: matrix.os == 'ubuntu-latest'
+      run: |
+        free -h
+        swapon --show
+        swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}')
+        echo "Found swap: $swap"
+        sudo swapoff $swap
+        # resize it (fallocate)
+        sudo fallocate -l 10G $swap
+        sudo mkswap $swap
+        sudo swapon $swap
+        free -h
+        (
+          while sleep 60; do
+            free -h
+          done
+        ) &
     - run: nix --experimental-features 'nix-command flakes' flake check -L
 
   # Steps to test CI automation in your own fork.
@@ -79,6 +96,7 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+        cachixArgs: '-v'
     - id: prepare-installer
       run: scripts/prepare-installer-for-github-actions
 
@@ -175,7 +193,21 @@ jobs:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
       - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes
+      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user
+
+  meson_build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      # Only meson packages that don't have a tests.run derivation.
+      # Those that have it are already built and tested as part of nix flake check.
+      - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g')
 
   flake_regressions:
     needs: vm_tests

.github/workflows/hydra_status.yml

@@ -1,20 +0,0 @@
-name: Hydra status
-
-permissions: read-all
-
-on:
-  schedule:
-    - cron: "12,42 * * * *"
-  workflow_dispatch:
-
-jobs:
-  check_hydra_status:
-    name: Check Hydra status
-    if: github.repository_owner == 'NixOS'
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - run: bash scripts/check-hydra-status.sh
-

.gitignore

@@ -54,6 +54,9 @@ perl/Makefile.config
 # /src/libfetchers
 /tests/unit/libfetchers/libnixfetchers-tests
 
+# /src/libflake
+/tests/unit/libflake/libnixflake-tests
+
 # /src/libstore/
 *.gen.*
 /src/libstore/tests

Makefile

@@ -18,6 +18,7 @@ makefiles = \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
+  src/libflake/local.mk \
   src/libcmd/local.mk \
   src/nix/local.mk \
   src/libutil-c/local.mk \
@@ -45,7 +46,8 @@ makefiles += \
   tests/unit/libstore-support/local.mk \
   tests/unit/libfetchers/local.mk \
   tests/unit/libexpr/local.mk \
-  tests/unit/libexpr-support/local.mk
+  tests/unit/libexpr-support/local.mk \
+  tests/unit/libflake/local.mk
 endif
 
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
@@ -90,7 +92,7 @@ ifdef HOST_WINDOWS
   GLOBAL_LDFLAGS += -Wl,--export-all-symbols
 endif
 
-GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src
+GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -include $(buildprefix)config.h -std=c++2a -I src
 
 # Include the main lib, causing rules to be defined
 

build-utils-meson/deps-lists/meson.build

@@ -0,0 +1,36 @@
+# These are private dependencies with pkg-config files. What private
+# means is that the dependencies are used by the library but they are
+# *not* used (e.g. `#include`-ed) in any installed header file, and only
+# in regular source code (`*.cc`) or private, uninstalled headers. They
+# are thus part of the *implementation* of the library, but not its
+# *interface*.
+#
+# See `man pkg-config` for some details.
+deps_private = [ ]
+
+# These are public dependencies with pkg-config files. Public is the
+# opposite of private: these dependencies are used in installed header
+# files. They are part of the interface (and implementation) of the
+# library.
+#
+# N.B. This concept is mostly unrelated to our own concept of a public
+# (stable) API, for consumption outside of the Nix repository.
+# `libnixutil` is an unstable C++ library, whose public interface is
+# likewise unstable. `libutilc` conversely is a hopefully-soon stable
+# C library, whose public interface --- including public but not private
+# dependencies --- will also likewise soon be stable.
+#
+# N.B. For distributions that care about "ABI" stability and not just
+# "API" stability, the private dependencies also matter as they can
+# potentially affect the public ABI.
+deps_public = [ ]
+
+# These are subproject deps (type == "internal"). They are other
+# packages in `/src` in this repo. The private vs public distinction is
+# the same as above.
+deps_private_subproject = [ ]
+deps_public_subproject = [ ]
+
+# These are dependencencies without pkg-config files. Ideally they are
+# just private, but they may also be public (e.g. boost).
+deps_other = [ ]

build-utils-meson/diagnostics/meson.build

@@ -0,0 +1,13 @@
+add_project_arguments(
+  '-Wno-deprecated-declarations',
+  '-Wimplicit-fallthrough',
+  '-Werror=switch',
+  '-Werror=switch-enum',
+  '-Werror=unused-result',
+  '-Wdeprecated-copy',
+  '-Wignored-qualifiers',
+  # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
+  # at ~1% overhead in `nix search`.
+  #
+  language : 'cpp',
+)

build-utils-meson/export-all-symbols/meson.build

@@ -0,0 +1,11 @@
+if host_machine.system() == 'cygwin' or host_machine.system() == 'windows'
+  # Windows DLLs are stricter about symbol visibility than Unix shared
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
+  # This is a temporary sledgehammer to export everything like on Unix,
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  linker_export_flags = ['-Wl,--export-all-symbols']
+else
+  linker_export_flags = []
+endif

build-utils-meson/export/meson.build

@@ -0,0 +1,30 @@
+requires_private = []
+foreach dep : deps_private_subproject
+  requires_private  += dep.name()
+endforeach
+requires_private += deps_private
+
+requires_public  = []
+foreach dep : deps_public_subproject
+  requires_public  += dep.name()
+endforeach
+requires_public += deps_public
+
+import('pkgconfig').generate(
+  this_library,
+  filebase : meson.project_name(),
+  name : 'Nix',
+  description : 'Nix Package Manager',
+  subdirs : ['nix'],
+  extra_cflags : ['-std=c++2a'],
+  requires : requires_public,
+  requires_private : requires_private,
+  libraries_private : libraries_private,
+)
+
+meson.override_dependency(meson.project_name(), declare_dependency(
+  include_directories : include_dirs,
+  link_with : this_library,
+  compile_args : ['-std=c++2a'],
+  dependencies : deps_public_subproject + deps_public,
+))

build-utils-meson/generate-header/meson.build

@@ -0,0 +1,7 @@
+bash = find_program('bash', native: true)
+
+gen_header = generator(
+  bash,
+  arguments : [ '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ],
+  output : '@PLAINNAME@.gen.hh',
+)

build-utils-meson/subprojects/meson.build

@@ -0,0 +1,19 @@
+foreach maybe_subproject_dep : deps_private_maybe_subproject
+  if maybe_subproject_dep.type_name() == 'internal'
+    deps_private_subproject += maybe_subproject_dep
+    # subproject sadly no good for pkg-config module
+    deps_other += maybe_subproject_dep
+  else
+    deps_private += maybe_subproject_dep
+  endif
+endforeach
+
+foreach maybe_subproject_dep : deps_public_maybe_subproject
+  if maybe_subproject_dep.type_name() == 'internal'
+    deps_public_subproject += maybe_subproject_dep
+    # subproject sadly no good for pkg-config module
+    deps_other += maybe_subproject_dep
+  else
+    deps_public += maybe_subproject_dep
+  endif
+endforeach

build-utils-meson/threads/meson.build

@@ -0,0 +1,6 @@
+# This is only conditional to work around
+# https://github.com/mesonbuild/meson/issues/13293. It should be
+# unconditional.
+if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc')
+  deps_private += dependency('threads')
+endif

configure.ac

@@ -400,6 +400,11 @@ AS_CASE(["$enable_markdown"],
 PKG_CHECK_MODULES([LIBGIT2], [libgit2])
 
 
+# Look for toml11, a required dependency.
+AC_LANG_PUSH(C++)
+AC_CHECK_HEADER([toml.hpp], [], [AC_MSG_ERROR([toml11 is not found.])])
+AC_LANG_POP(C++)
+
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])
 

dep-patches/boehmgc-traceable_allocator-public.diff

@@ -1,12 +0,0 @@
-diff --git a/include/gc_allocator.h b/include/gc_allocator.h
-index 597c7f13..587286be 100644
---- a/include/gc_allocator.h
-+++ b/include/gc_allocator.h
-@@ -312,6 +312,7 @@ public:
- 
- template<>
- class traceable_allocator<void> {
-+public:
-   typedef size_t      size_type;
-   typedef ptrdiff_t   difference_type;
-   typedef void*       pointer;

doc/manual/generate-builtin-constants.nix

@@ -1,31 +0,0 @@
-let
-  inherit (builtins) concatStringsSep attrValues mapAttrs;
-  inherit (import <nix/utils.nix>) optionalString squash;
-in
-
-builtinsInfo:
-let
-  showBuiltin = name: { doc, type, impure-only }:
-    let
-      type' = optionalString (type != null) " (${type})";
-
-      impureNotice = optionalString impure-only ''
-        > **Note**
-        >
-        > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
-      '';
-    in
-    squash ''
-      <dt id="builtins-${name}">
-        <a href="#builtins-${name}"><code>${name}</code></a>${type'}
-      </dt>
-      <dd>
-
-      ${doc}
-
-      ${impureNotice}
-
-      </dd>
-    '';
-in
-concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo))

doc/manual/generate-builtins.nix

@@ -5,8 +5,10 @@ in
 
 builtinsInfo:
 let
-  showBuiltin = name: { doc, args, arity, experimental-feature }:
+  showBuiltin = name: { doc, type ? null, args ? [ ], experimental-feature ? null, impure-only ? false }:
     let
+      type' = optionalString (type != null) " (${type})";
+
       experimentalNotice = optionalString (experimental-feature != null) ''
         > **Note**
         >
@@ -18,18 +20,26 @@ let
         > extra-experimental-features = ${experimental-feature}
         > ```
       '';
+
+      impureNotice = optionalString impure-only ''
+        > **Note**
+        >
+        > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+      '';
     in
     squash ''
       <dt id="builtins-${name}">
-        <a href="#builtins-${name}"><code>${name} ${listArgs args}</code></a>
+        <a href="#builtins-${name}"><code>${name}${listArgs args}</code></a>${type'}
       </dt>
       <dd>
 
       ${experimentalNotice}
 
       ${doc}
+
+      ${impureNotice}
       </dd>
     '';
-  listArgs = args: concatStringsSep " " (map (s: "<var>${s}</var>") args);
+  listArgs = args: concatStringsSep "" (map (s: " <var>${s}</var>") args);
 in
 concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo))

doc/manual/generate-manpage.nix

@@ -116,9 +116,12 @@ let
             storeInfo = commandInfo.stores;
             inherit inlineHTML;
           };
+          hasInfix = infix: content:
+            builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content);
         in
         optionalString (details ? doc) (
-          if match ".*@store-types@.*" details.doc != null
+          # An alternate implementation with builtins.match stack overflowed on some systems.
+          if hasInfix "@store-types@" details.doc
           then help-stores
           else details.doc
         );

doc/manual/local.mk

@@ -140,16 +140,10 @@ $(d)/xp-features.json: $(doc_nix)
 
 $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix)
-	@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
-	@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
-	@mv $@.tmp $@
-
 $(d)/language.json: $(doc_nix)
 	$(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp
 	@mv $@.tmp $@
@@ -217,7 +211,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/redirects.js

@@ -1,7 +1,7 @@
 // redirect rules for URL fragments (client-side) to prevent link rot.
 // this must be done on the client side, as web servers do not see the fragment part of the URL.
 // it will only work with JavaScript enabled in the browser, but this is the best we can do here.
-// see ./_redirects for path redirects (client-side)
+// see src/_redirects for path redirects (server-side)
 
 // redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
@@ -238,12 +238,12 @@ const redirects = {
     "attr-system": "language/derivations.html#attr-system",
     "ssec-derivation": "language/derivations.html",
     "ch-expression-language": "language/index.html",
-    "sec-constructs": "language/constructs.html",
+    "sec-constructs": "language/syntax.html",
-    "sect-let-language": "language/constructs.html#let-language",
+    "sect-let-language": "language/syntax.html#let-expressions",
-    "ss-functions": "language/constructs.html#functions",
+    "ss-functions": "language/syntax.html#functions",
     "sec-language-operators": "language/operators.html",
     "table-operators": "language/operators.html",
-    "ssec-values": "language/values.html",
+    "ssec-values": "language/types.html",
     "gloss-closure": "glossary.html#gloss-closure",
     "gloss-derivation": "glossary.html#gloss-derivation",
     "gloss-deriver": "glossary.html#gloss-deriver",
@@ -335,11 +335,15 @@ const redirects = {
     "ssec-relnotes-2.2": "release-notes/rl-2.2.html",
     "ssec-relnotes-2.3": "release-notes/rl-2.3.html",
   },
-  "language/values.html": {
+  "language/types.html": {
     "simple-values": "#primitives",
     "lists": "#list",
     "strings": "#string",
     "attribute-sets": "#attribute-set",
+    "type-number": "#type-int",
+  },
+  "language/syntax.html": {
+    "scoping-rules": "scoping.html",
   },
   "installation/installing-binary.html": {
     "linux": "uninstall.html#linux",

doc/manual/rl-next/drop-vendored-toml11.md

@@ -0,0 +1,6 @@
+---
+synopsis: Stop vendoring toml11
+---
+
+We don't apply any patches to it, and vendoring it locks users into
+bugs (it hasn't been updated since its introduction in late 2021).

doc/manual/rl-next/harden-user-sandboxing.md

@@ -0,0 +1,7 @@
+---
+synopsis: Harden the user sandboxing
+significance: significant
+issues:
+---
+
+The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user.

doc/manual/rl-next/nix-shell-looks-for-shell-nix.md

@@ -0,0 +1,28 @@
+---
+synopsis: "`nix-shell <directory>` looks for `shell.nix`"
+significance: significant
+issues:
+- 496
+- 2279
+- 4529
+- 5431
+- 11053
+prs:
+- 11057
+---
+
+`nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory.
+
+Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue.
+This adjustment addresses a commonly reported problem.
+
+This also applies to `nix-shell` shebang scripts. Consider the following example:
+
+```shell
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash
+```
+
+This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise.
+
+The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`.

doc/manual/rl-next/repl-doc-renders-doc-comments.md

@@ -0,0 +1,53 @@
+---
+synopsis: "`nix-repl`'s `:doc` shows documentation comments"
+significance: significant
+issues:
+- 3904
+- 10771
+prs:
+- 1652
+- 9054
+- 11072
+---
+
+`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions.
+This feature has been extended to also render function documentation comments, in accordance with [RFC 145].
+
+Example:
+
+```
+nix-repl> :doc lib.toFunction
+Function toFunction
+    … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5
+
+    Turns any non-callable values into constant functions. Returns
+    callable values as is.
+
+Inputs
+
+    v
+
+      : Any value
+
+Examples
+
+    :::{.example}
+
+## lib.trivial.toFunction usage example
+
+      | nix-repl> lib.toFunction 1 2
+      | 1
+      | 
+      | nix-repl> lib.toFunction (x: x + 1) 2
+      | 3
+
+    :::
+```
+
+Known limitations:
+- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`.
+- Some extensions to markdown are not yet supported, as you can see in the example above.
+
+We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145].
+
+[RFC 145]: https://github.com/NixOS/rfcs/pull/145

doc/manual/rl-next/shebang-relative.md

@@ -0,0 +1,62 @@
+---
+synopsis: "`nix-shell` shebang uses relative path"
+prs:
+- 5088
+- 11058
+issues:
+- 4232
+---
+
+<!-- unfortunately no link target for the specific syntax -->
+Relative [path](@docroot@/language/values.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary?highlight=base%20directory#gloss-base-directory).
+Previously they were resolved relative to the current working directory.
+
+For example, consider the following script in `~/myproject/say-hi`:
+
+```shell
+#!/usr/bin/env nix-shell
+#!nix-shell --expr 'import ./shell.nix'
+#!nix-shell --arg toolset './greeting-tools.nix'
+#!nix-shell -i bash
+hello
+```
+
+Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example:
+
+```console
+[hostname:~]$ ./myproject/say-hi
+error:
+       … while calling the 'import' builtin
+         at «string»:1:2:
+            1| (import ./shell.nix)
+             |  ^
+
+       error: path '/home/user/shell.nix' does not exist
+```
+
+Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used.
+
+```console
+$ ./myproject/say-hi
+Hello, world!
+```
+
+**Opt-out**
+
+This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update.
+The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`.
+This option will be removed in a future release.
+
+**`nix` command shebang**
+
+The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner.
+
+Example:
+
+```shell
+#!/usr/bin/env nix
+#!nix develop
+#!nix --expr ``import ./shell.nix``
+#!nix -c bash
+hello
+```

doc/manual/src/SUMMARY.md.in

@@ -25,17 +25,17 @@
   - [Store Types](store/types/index.md)
 {{#include ./store/types/SUMMARY.md}}
 - [Nix Language](language/index.md)
-  - [Data Types](language/values.md)
+  - [Data Types](language/types.md)
-  - [Language Constructs](language/constructs.md)
+    - [String context](language/string-context.md)
+  - [Syntax and semantics](language/syntax.md)
+    - [Scoping rules](language/scope.md)
     - [String interpolation](language/string-interpolation.md)
     - [Lookup path](language/constructs/lookup-path.md)
-    - [String context](language/string-context.md)
   - [Operators](language/operators.md)
+  - [Built-ins](language/builtins.md)
     - [Derivations](language/derivations.md)
       - [Advanced Attributes](language/advanced-attributes.md)
       - [Import From Derivation](language/import-from-derivation.md)
-  - [Built-in Constants](language/builtin-constants.md)
-  - [Built-in Functions](language/builtins.md)
 - [Package Management](package-management/index.md)
   - [Profiles](package-management/profiles.md)
   - [Garbage Collection](package-management/garbage-collection.md)

doc/manual/src/_redirects

@@ -1,5 +1,5 @@
 # redirect rules for paths (server-side) to prevent link rot.
-# see ./redirects.js for redirects based on URL fragments (client-side)
+# see ../redirects.js for redirects based on URL fragments (client-side)
 #
 # concrete user story this supports:
 # - user finds URL to the manual for Nix x.y
@@ -27,6 +27,9 @@
 /expressions/language-operators /language/operators 301!
 /expressions/language-values /language/values 301!
 /expressions/* /language/:splat 301!
+/language/values /language/types 301!
+/language/constructs /language/syntax 301!
+/language/builtin-constants /language/builtins 301!
 
 /installation/installation /installation 301!
 

doc/manual/src/command-ref/env-common.md

@@ -10,7 +10,7 @@ Most Nix commands interpret the following environment variables:
 - <span id="env-NIX_PATH">[`NIX_PATH`](#env-NIX_PATH)</span>
 
   A colon-separated list of directories used to look up the location of Nix
-  expressions using [paths](@docroot@/language/values.md#type-path)
+  expressions using [paths](@docroot@/language/types.md#type-path)
   enclosed in angle brackets (i.e., `<path>`),
   e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the
   [`-I` option](@docroot@/command-ref/opt-common.md#opt-I).

doc/manual/src/command-ref/nix-env/install.md

@@ -57,7 +57,7 @@ The arguments *args* map to store paths in a number of possible ways:
   easy way to copy user environment elements from one profile to
   another.
 
-- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/constructs.md#functions) that are called with the [default Nix expression] as their single argument.
+- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the [default Nix expression] as their single argument.
   The derivations returned by those function calls are installed.
   This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name.
 

doc/manual/src/command-ref/opt-common.md

@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options:
 
   This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`.
   When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters.
-  It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ?  defaultValue }: ...`).
+  It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ?  defaultValue }: ...`).
 
   With `--arg`, you can also call functions that have arguments without a default value (or override a default value).
   That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*.

doc/manual/src/contributing/hacking.md

@@ -122,7 +122,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/
 
   The docs can take a while to build, so you may want to disable this for local development.
 - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests.
-- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests.
 - `OPTIMIZE=1` to enable optimizations.
 - `libraries=libutil programs=` to only build a specific library.
 

doc/manual/src/contributing/testing.md

@@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
 > …
 > ```
 
-The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`.
+The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`.
-Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`.
+Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`.
 
 Data for unit tests is stored in a `data` subdir of the directory for each unit test executable.
-For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`.
+For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`.
-The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
 Note that each executable only gets the data for its tests.
 
-The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`.
+The unit test libraries are in `src/${library_name_without-nix}-test-support`.
 All headers are in a `tests` subdirectory so they are included with `#include "tests/"`.
 
 The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing.
@@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick
 
 ### Running tests
 
-You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`.
+You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`.
-Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`.
+A environment variables that Google Test accepts are also worth knowing:
+
+1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests)
+
+   This is used for finer-grained filtering of which tests to run.
+
+
+2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes)
+
+   This is used to avoid logging passing tests.
+
+Putting the two together, one might run
+
+```bash
+GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v
+```
+
+for short but comprensive output.
 
 ### Characterisation testing { #characaterisation-testing-unit }
 
@@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional)
 Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used.
 For example:
 ```shell-session
-$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN
+$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v
 ...
 [  SKIPPED ] WorkerProtoTest.string_read
 [  SKIPPED ] WorkerProtoTest.string_write
@@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all.
 The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.
 Each test is a bash script.
 
+Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests.
+
 ### Running the whole test suite
 
 The whole test suite can be run with:
@@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review.
 
 To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`.
 
+### Running functional tests on NixOS
+
+We run the functional tests not just in the build, but also in VM tests.
+This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment.
+
+The recommended way to run these tests during development is:
+
+```shell
+nix build .#hydraJobs.tests.functional_user.quickBuild
+```
+
+The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`.
+
+Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization.
+
 ## Integration tests
 
 The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
 These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
 Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
 
-You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
+You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`.
+
+If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`.
 
 ## Installer tests
 

doc/manual/src/glossary.md

@@ -312,7 +312,7 @@
 
 - [package attribute set]{#package-attribute-set}
 
-  An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as
+  An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as
   - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output),
   - attributes that declare something about how the package is supposed to be installed or used,
   - other metadata or arbitrary attributes.
@@ -325,9 +325,9 @@
 
   See [String interpolation](./language/string-interpolation.md) for details.
 
-  [string]: ./language/values.md#type-string
+  [string]: ./language/types.md#type-string
-  [path]: ./language/values.md#type-path
+  [path]: ./language/types.md#type-path
-  [attribute name]: ./language/values.md#attribute-set
+  [attribute name]: ./language/types.md#attribute-set
 
 - [base directory]{#gloss-base-directory}
 

doc/manual/src/language/advanced-attributes.md

@@ -302,6 +302,12 @@ Derivations can declare some infrequently used optional attributes.
     (associative) arrays. For example, the attribute `hardening.format = true`
     ends up as the Bash associative array element `${hardening[format]}`.
 
+    > **Warning**
+    >
+    > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites),
+    [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize.
+    will have no effect.
+
   - [`outputChecks`]{#adv-attr-outputChecks}\
     When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks`
     attribute allows defining checks per-output.

doc/manual/src/language/builtin-constants-prefix.md

@@ -1,5 +0,0 @@
-# Built-in Constants
-
-These constants are built into the Nix language evaluator:
-
-<dl>

doc/manual/src/language/builtin-constants-suffix.md

@@ -1 +0,0 @@
-</dl>

doc/manual/src/language/builtins-prefix.md

@@ -1,9 +1,11 @@
-# Built-in Functions
+# Built-ins
 
-This section lists the functions built into the Nix language evaluator.
+This section lists the values and functions built into the Nix language evaluator.
-All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant.
+All built-ins are available through the global [`builtins`](#builtins-builtins) constant.
 
-For convenience, some built-ins can be accessed directly:
+Some built-ins are also exposed directly in the global scope:
+
+<!-- TODO(@rhendric, #10970): this list is incomplete -->
 
 - [`derivation`](#builtins-derivation)
 - [`import`](#builtins-import)

doc/manual/src/language/constructs/lookup-path.md

@@ -4,9 +4,9 @@
 >
 > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>`
 
-A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry.
+A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry.
 
-The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
+The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath).
 
 See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution.
 

doc/manual/src/language/derivations.md

@@ -12,7 +12,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
 
 ### Required
 
-- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string))
+- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string))
 
   A symbolic name for the derivation.
   It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path).
@@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
   > The store derivation's path will be `/nix/store/<hash>-hello.drv`.
   > The [output](#attr-outputs) paths will be of the form `/nix/store/<hash>-hello[-<output>]`
 
-- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string))
+- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string))
 
   The system type on which the [`builder`](#attr-builder) executable is meant to be run.
 
@@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
   > }
   > ```
   >
-  > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation.
+  > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation.
 
-- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string))
+- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string))
 
   Path to an executable that will perform the build.
 
@@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
 
 ### Optional
 
-- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string))
+- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string))
 
   Default: `[ ]`
 
@@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
   > };
   > ```
 
-- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string))
+- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string))
 
   Default: `[ "out" ]`
 

doc/manual/src/language/index.md

@@ -53,7 +53,7 @@ This is an incomplete overview of language features, by example.
   <td>
 
 
-   *Basic values ([primitives](@docroot@/language/values.md#primitives))*
+   *Basic values ([primitives](@docroot@/language/types.md#primitives))*
 
 
   </td>
@@ -71,7 +71,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [string](@docroot@/language/values.md#type-string)
+   A [string](@docroot@/language/types.md#type-string)
 
   </td>
  </tr>
@@ -102,7 +102,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [comment](@docroot@/language/constructs.md#comments).
+   A [comment](@docroot@/language/syntax.md#comments).
 
   </td>
  </tr>
@@ -130,7 +130,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   [Booleans](@docroot@/language/values.md#type-boolean)
+   [Booleans](@docroot@/language/types.md#type-boolean)
 
   </td>
  </tr>
@@ -142,7 +142,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   [Null](@docroot@/language/values.md#type-null) value
+   [Null](@docroot@/language/types.md#type-null) value
 
   </td>
  </tr>
@@ -154,7 +154,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   An [integer](@docroot@/language/values.md#type-number)
+   An [integer](@docroot@/language/types.md#type-int)
 
   </td>
  </tr>
@@ -166,7 +166,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [floating point number](@docroot@/language/values.md#type-number)
+   A [floating point number](@docroot@/language/types.md#type-float)
 
   </td>
  </tr>
@@ -178,7 +178,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   An absolute [path](@docroot@/language/values.md#type-path)
+   An absolute [path](@docroot@/language/types.md#type-path)
 
   </td>
  </tr>
@@ -190,7 +190,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [path](@docroot@/language/values.md#type-path) relative to the file containing this Nix expression
+   A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression
 
   </td>
  </tr>
@@ -202,7 +202,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A home [path](@docroot@/language/values.md#type-path). Evaluates to the `"<user's home directory>/.config"`.
+   A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"<user's home directory>/.config"`.
 
   </td>
  </tr>
@@ -238,7 +238,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   An [attribute set](@docroot@/language/values.md#attribute-set) with attributes named `x` and `y`
+   An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y`
 
   </td>
  </tr>
@@ -262,7 +262,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [recursive set](@docroot@/language/constructs.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`.
+   A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`.
 
   </td>
  </tr>
@@ -278,7 +278,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   [Lists](@docroot@/language/values.md#list) with three elements.
+   [Lists](@docroot@/language/types.md#list) with three elements.
 
   </td>
  </tr>
@@ -362,7 +362,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   [Attribute selection](@docroot@/language/values.md#attribute-set) (evaluates to `1`)
+   [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`)
 
   </td>
  </tr>
@@ -374,7 +374,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   [Attribute selection](@docroot@/language/values.md#attribute-set) with default (evaluates to `3`)
+   [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`)
 
   </td>
  </tr>
@@ -410,7 +410,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   [Conditional expression](@docroot@/language/constructs.md#conditionals).
+   [Conditional expression](@docroot@/language/syntax.md#conditionals).
 
   </td>
  </tr>
@@ -422,7 +422,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   [Assertion](@docroot@/language/constructs.md#assertions) check (evaluates to `"yes!"`).
+   [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`).
 
   </td>
  </tr>
@@ -434,7 +434,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   Variable definition. See [`let`-expressions](@docroot@/language/constructs.md#let-expressions).
+   Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions).
 
   </td>
  </tr>
@@ -448,7 +448,7 @@ This is an incomplete overview of language features, by example.
 
    Add all attributes from the given set to the scope (evaluates to `1`).
 
-   See [`with`-expressions](@docroot@/language/constructs.md#with-expressions) for details and shadowing caveats.
+   See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats.
 
   </td>
  </tr>
@@ -462,7 +462,7 @@ This is an incomplete overview of language features, by example.
 
    Adds the variables to the current scope (attribute set or `let` binding).
    Desugars to `pkgs = pkgs; src = src;`.
-   See [Inheriting attributes](@docroot@/language/constructs.md#inheriting-attributes).
+   See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes).
 
   </td>
  </tr>
@@ -476,14 +476,14 @@ This is an incomplete overview of language features, by example.
 
    Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding).
    Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`.
-   See [Inheriting attributes](@docroot@/language/constructs.md#inheriting-attributes).
+   See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes).
 
   </td>
  </tr>
  <tr>
   <td>
 
-   *[Functions](@docroot@/language/constructs.md#functions) (lambdas)*
+   *[Functions](@docroot@/language/syntax.md#functions) (lambdas)*
 
   </td>
   <td>
@@ -500,7 +500,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [function](@docroot@/language/constructs.md#functions) that expects an integer and returns it increased by 1.
+   A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1.
 
   </td>
  </tr>
@@ -512,7 +512,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   Curried [function](@docroot@/language/constructs.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum.
+   Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum.
 
   </td>
  </tr>
@@ -524,7 +524,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [function](@docroot@/language/constructs.md#functions) call (evaluates to 101)
+   A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101)
 
   </td>
  </tr>
@@ -536,7 +536,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [function](@docroot@/language/constructs.md#functions) bound to a variable and subsequently called by name (evaluates to 103)
+   A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103)
 
   </td>
  </tr>
@@ -548,7 +548,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y` and concatenates them
+   A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them
 
   </td>
  </tr>
@@ -560,7 +560,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [function](@docroot@/language/constructs.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y`
+   A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y`
 
   </td>
  </tr>
@@ -572,7 +572,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes
+   A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes
 
   </td>
  </tr>
@@ -586,7 +586,7 @@ This is an incomplete overview of language features, by example.
   </td>
   <td>
 
-   A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args`
+   A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args`
 
   </td>
  </tr>

doc/manual/src/language/operators.md

@@ -27,11 +27,11 @@
 | Logical disjunction (`OR`)             | *bool* <code>\|\|</code> *bool*            | left          | 13         |
 | [Logical implication]                  | *bool* `->` *bool*                         | right         | 14         |
 
-[string]: ./values.md#type-string
+[string]: ./types.md#type-string
-[path]: ./values.md#type-path
+[path]: ./types.md#type-path
-[number]: ./values.md#type-number
+[number]: ./types.md#type-float <!-- TODO(@rhendric, #10970): rationalize this -->
-[list]: ./values.md#list
+[list]: ./types.md#list
-[attribute set]: ./values.md#attribute-set
+[attribute set]: ./types.md#attribute-set
 
 ## Attribute selection
 
@@ -42,7 +42,7 @@
 Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*.
 If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation.
 
-An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set).
+An attribute path is a dot-separated list of [attribute names](./types.md#attribute-set).
 
 > **Syntax**
 >
@@ -61,7 +61,7 @@ The result is a [Boolean] value.
 
 See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr)
 
-[Boolean]: ./values.md#type-boolean
+[Boolean]: ./types.md#type-boolean
 
 [Has attribute]: #has-attribute
 
@@ -172,7 +172,7 @@ All comparison operators are implemented in terms of `<`, and the following equi
 - Numbers are type-compatible, see [arithmetic] operators.
 - Floating point numbers only differ up to a limited precision.
 
-[function]: ./constructs.md#functions
+[function]: ./syntax.md#functions
 
 [Equality]: #equality
 

doc/manual/src/language/scope.md

@@ -0,0 +1,14 @@
+# Scoping rules
+
+Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules.
+
+* primary scope: explicitly-bound variables
+  * [`let`](./syntax.md#let-expressions)
+  * [`inherit`](./syntax.md#inheriting-attributes)
+  * [function](./syntax.md#functions) arguments
+
+* secondary scope: implicitly-bound variables
+  * [`with`](./syntax.md#with-expressions)
+
+Primary scope takes precedence over secondary scope.
+See [`with`](./syntax.md#with-expressions) for a detailed example.

doc/manual/src/language/string-context.md

@@ -111,7 +111,7 @@ It creates an [attribute set] representing the string context, which can be insp
 
 [`builtins.hasContext`]: ./builtins.md#builtins-hasContext
 [`builtins.getContext`]: ./builtins.md#builtins-getContext
-[attribute set]: ./values.md#attribute-set
+[attribute set]: ./types.md#attribute-set
 
 ## Clearing string contexts
 

doc/manual/src/language/string-interpolation.md

@@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu
 
 Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression).
 
-[string]: ./values.md#type-string
+[string]: ./types.md#type-string
-[path]: ./values.md#type-path
+[path]: ./types.md#type-path
-[attribute set]: ./values.md#attribute-set
+[attribute set]: ./types.md#attribute-set
 
 ## Examples
 
@@ -43,6 +43,47 @@ configureFlags = "
 Note that Nix expressions and strings can be arbitrarily nested;
 in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`).
 
+To write a literal `${` in an regular string, escape it with a backslash (`\`).
+
+> **Example**
+>
+> ```nix
+> "echo \${PATH}"
+> ```
+>
+>     "echo ${PATH}"
+
+To write a literal `${` in an indented string, escape it with two single quotes (`''`).
+
+> **Example**
+>
+> ```nix
+> ''
+>   echo ''${PATH}
+> ''
+> ```
+>
+>     "echo ${PATH}\n"
+
+`$${` can be written literally in any string.
+
+> **Example**
+>
+> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References).
+> This can be expressed directly in the Nix language strings:
+>
+> ```nix
+> ''
+>   MAKEVAR = Hello
+>   all:
+>   	@export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR}
+> ''
+> ```
+>
+>     "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n"
+
+See the [documentation on strings][string] for details.
+
 ### Path
 
 Rather than writing

doc/manual/src/language/syntax.md

@@ -1,8 +1,363 @@
 # Language Constructs
 
+This section covers syntax and semantics of the Nix language.
+
+## Basic Literals
+
+### String {#string-literal}
+
+  *Strings* can be written in three ways.
+
+  The most common way is to enclose the string between double quotes, e.g., `"foo bar"`.
+  Strings can span multiple lines.
+  The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation].
+
+  [string interpolation]: ./string-interpolation.md
+
+  The following must be escaped to represent them within a string, by prefixing with a backslash (`\`):
+
+  - Double quote (`"`)
+
+    > **Example**
+    >
+    > ```nix
+    > "\""
+    > ```
+    >
+    >    "\""
+
+  - Backslash (`\`)
+
+    > **Example**
+    >
+    > ```nix
+    > "\\"
+    > ```
+    >
+    >    "\\"
+
+  - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly"
+
+    > **Example**
+    >
+    > ```nix
+    > "\${"
+    > ```
+    >
+    >    "\${"
+
+  The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively.
+
+  A "double-dollar-curly" (`$${`) can be written literally.
+
+  > **Example**
+  >
+  > ```nix
+  > "$${"
+  > ```
+  >
+  >    "$\${"
+
+  String values are output on the terminal with Nix-specific escaping.
+  Strings written to files will contain the characters encoded by the escaping.
+
+  The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so:
+
+  ```nix
+  ''
+    This is the first line.
+    This is the second line.
+      This is the third line.
+  ''
+  ```
+
+  This kind of string literal intelligently strips indentation from
+  the start of each line. To be precise, it strips from each line a
+  number of spaces equal to the minimal indentation of the string as a
+  whole (disregarding the indentation of empty lines). For instance,
+  the first and second line are indented two spaces, while the third
+  line is indented four spaces. Thus, two spaces are stripped from
+  each line, so the resulting string is
+
+  ```nix
+  "This is the first line.\nThis is the second line.\n  This is the third line.\n"
+  ```
+
+  > **Note**
+  >
+  > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line.
+
+  > **Warning**
+  >
+  > Prefixed tab characters are not stripped.
+  >
+  > > **Example**
+  > >
+  > > The following indented string is prefixed with tabs:
+  > >
+  > > ''
+  > > 	all:
+  > > 		@echo hello
+  > > ''
+  > >
+  > >     "\tall:\n\t\t@echo hello\n"
+
+  Indented strings support [string interpolation].
+
+  The following must be escaped to represent them in an indented string:
+
+  - `$` is escaped by prefixing it with two single quotes (`''`)
+
+    > **Example**
+    >
+    > ```nix
+    > ''
+    >   ''$
+    > ''
+    > ```
+    >
+    >     "$\n"
+
+  - `''` is escaped by prefixing it with one single quote (`'`)
+
+    > **Example**
+    >
+    > ```nix
+    > ''
+    >   '''
+    > ''
+    > ```
+    >
+    >     "''\n"
+
+  These special characters are escaped as follows:
+  - Linefeed (`\n`): `''\n`
+  - Carriage return (`\r`): `''\r`
+  - Tab (`\t`): `''\t`
+
+  `''\` escapes any other character.
+
+  A "double-dollar-curly" (`$${`) can be written literally.
+
+  > **Example**
+  >
+  > ```nix
+  > ''
+  >   $${
+  > ''
+  > ```
+  >
+  >     "$\${\n"
+
+  Indented strings are primarily useful in that they allow multi-line
+  string literals to follow the indentation of the enclosing Nix
+  expression, and that less escaping is typically necessary for
+  strings representing languages such as shell scripts and
+  configuration files because `''` is much less common than `"`.
+  Example:
+
+  ```nix
+  stdenv.mkDerivation {
+    ...
+    postInstall =
+      ''
+        mkdir $out/bin $out/etc
+        cp foo $out/bin
+        echo "Hello World" > $out/etc/foo.conf
+        ${if enableBar then "cp bar $out/bin" else ""}
+      '';
+    ...
+  }
+  ```
+
+  Finally, as a convenience, *URIs* as defined in appendix B of
+  [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as
+  is*, without quotes. For instance, the string
+  `"http://example.org/foo.tar.bz2"` can also be written as
+  `http://example.org/foo.tar.bz2`.
+
+### Number {#number-literal}
+
+<!-- TODO(@rhendric, #10970): split this into int and float -->
+
+  Numbers, which can be *integers* (like `123`) or *floating point*
+  (like `123.43` or `.27e13`).
+
+  See [arithmetic] and [comparison] operators for semantics.
+
+  [arithmetic]: ./operators.md#arithmetic
+  [comparison]: ./operators.md#comparison
+
+### Path {#path-literal}
+
+  *Paths* can be expressed by path literals such as `./builder.sh`.
+
+  A path literal must contain at least one slash to be recognised as such.
+  For instance, `builder.sh` is not a path:
+  it's parsed as an expression that selects the attribute `sh` from the variable `builder`.
+
+  Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory).
+  Path literals may also refer to absolute paths by starting with a slash.
+
+  > **Note**
+  >
+  > Absolute paths make expressions less portable.
+  > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead.
+  > This avoids some confusion about whether files at that location will be used during evaluation.
+  > It also avoids unintentional situations where some function might try to copy everything at the location into the store.
+
+  If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory.
+  For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`.
+  Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation.
+
+  Path literals can also include [string interpolation], besides being [interpolated into other expressions].
+
+  [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions
+
+  At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.
+
+  `a.${foo}/b.${bar}` is a syntactically valid number division operation.
+  `./a.${foo}/b.${bar}` is a path.
+
+  [Lookup path](./constructs/lookup-path.md) literals such as `<nixpkgs>` also resolve to path values.
+
+## List {#list-literal}
+
+Lists are formed by enclosing a whitespace-separated list of values
+between square brackets. For example,
+
+```nix
+[ 123 ./foo.nix "abc" (f { x = y; }) ]
+```
+
+defines a list of four elements, the last being the result of a call to
+the function `f`. Note that function calls have to be enclosed in
+parentheses. If they had been omitted, e.g.,
+
+```nix
+[ 123 ./foo.nix "abc" f { x = y; } ]
+```
+
+the result would be a list of five elements, the fourth one being a
+function and the fifth being a set.
+
+Note that lists are only lazy in values, and they are strict in length.
+
+Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt).
+
+## Attribute Set {#attrs-literal}
+
+An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`).
+
+An attribute name can be an identifier or a [string](#string).
+An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`).
+
+> **Syntax**
+>
+> *name* = *identifier* | *string* \
+> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*`
+
+Names and values are separated by an equal sign (`=`).
+Each value is an arbitrary expression terminated by a semicolon (`;`).
+
+> **Syntax**
+>
+> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}`
+
+Attributes can appear in any order.
+An attribute name may only occur once.
+
+Example:
+
+```nix
+{
+  x = 123;
+  text = "Hello";
+  y = f { bla = 456; };
+}
+```
+
+This defines a set with attributes named `x`, `text`, `y`.
+
+Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection).
+
+Example:
+
+```nix
+{ a = "Foo"; b = "Bar"; }.a
+```
+
+This evaluates to `"Foo"`.
+
+It is possible to provide a default value in an attribute selection using the `or` keyword.
+
+Example:
+
+```nix
+{ a = "Foo"; b = "Bar"; }.c or "Xyzzy"
+```
+
+```nix
+{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy"
+```
+
+will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set.
+
+You can use arbitrary double-quoted strings as attribute names:
+
+```nix
+{ "$!@#?" = 123; }."$!@#?"
+```
+
+```nix
+let bar = "bar"; in
+{ "foo ${bar}" = 123; }."foo ${bar}"
+```
+
+Both will evaluate to `123`.
+
+Attribute names support [string interpolation]:
+
+```nix
+let bar = "foo"; in
+{ foo = 123; }.${bar}
+```
+
+```nix
+let bar = "foo"; in
+{ ${bar} = 123; }.foo
+```
+
+Both will evaluate to `123`.
+
+In the special case where an attribute name inside of a set declaration
+evaluates to `null` (which is normally an error, as `null` cannot be coerced to
+a string), that attribute is simply not added to the set:
+
+```nix
+{ ${if foo then "bar" else null} = true; }
+```
+
+This will evaluate to `{}` if `foo` evaluates to `false`.
+
+A set that has a `__functor` attribute whose value is callable (i.e. is
+itself a function or a set with a `__functor` attribute whose value is
+callable) can be applied as if it were a function, with the set itself
+passed in first , e.g.,
+
+```nix
+let add = { __functor = self: x: x + self.x; };
+    inc = add // { x = 1; };
+in inc 1
+```
+
+evaluates to `2`. This can be used to attach metadata to a function
+without the caller needing to treat it specially, or to implement a form
+of object-oriented programming, for example.
+
 ## Recursive sets
 
-Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other.
+Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other.
 
 > *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}`
 
@@ -54,7 +409,7 @@ This evaluates to `"foobar"`.
 
 ## Inheriting attributes
 
-When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes).
+When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes).
 This can be shortened using the `inherit` keyword.
 
 Example:
@@ -470,17 +825,3 @@ Does evaluate to `"inner"`.
   > ```console
   > 1
   > ```
-
-## Scoping rules
-
-Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules.
-
-* primary scope --- explicitly-bound variables
-  * [`let`](#let-expressions)
-  * [`inherit`](#inheriting-attributes)
-  * function arguments
-
-* secondary scope --- implicitly-bound variables
-  * [`with`](#with-expressions)
-Primary scope takes precedence over secondary scope.
-See [`with`](#with-expressions) for a detailed example.

doc/manual/src/language/types.md

@@ -0,0 +1,120 @@
+# Data Types
+
+Every value in the Nix language has one of the following types:
+
+* [Integer](#type-int)
+* [Float](#type-float)
+* [Boolean](#type-bool)
+* [String](#type-string)
+* [Path](#type-path)
+* [Null](#type-null)
+* [Attribute set](#type-attrs)
+* [List](#type-list)
+* [Function](#type-function)
+* [External](#type-external)
+
+## Primitives
+
+### Integer {#type-int}
+
+An _integer_ in the Nix language is a signed 64-bit integer.
+
+Non-negative integers can be expressed as [integer literals](syntax.md#number-literal).
+Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic).
+The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer.
+
+### Float {#type-float}
+
+A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number.
+
+Most non-negative floats can be expressed as [float literals](syntax.md#number-literal).
+Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic).
+The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float.
+
+### Boolean {#type-bool}
+
+A _boolean_ in the Nix language is one of _true_ or _false_.
+
+<!-- TODO: mention the top-level environment -->
+
+These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false).
+The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean.
+
+### String {#type-string}
+
+A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md).
+Nix does not assume or support working natively with character encodings.
+
+String values without string context can be expressed as [string literals](syntax.md#string-literal).
+The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string.
+
+### Path {#type-path}
+
+A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path.
+Path values are distinct from string values, even if they contain the same sequence of bytes.
+Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution.
+
+[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html
+
+Paths are suitable for referring to local files, and are often preferable over strings.
+- Path values do not contain trailing or duplicate slashes, `.`, or `..`.
+- Relative path literals are automatically resolved relative to their [base directory].
+- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file.
+
+[base directory]: @docroot@/glossary.md#gloss-base-directory
+
+A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store.
+For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/<hash>-foo.txt"`.
+Operations such as [`import`] can also expect a path to resolve to a readable file or directory.
+
+[string interpolation]: string-interpolation.md#interpolated-expression
+[string-and-path concatenation]: operators.md#string-and-path-concatenation
+[`import`]: builtins.md#builtins-import
+
+> **Note**
+>
+> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression.
+> For example, assume you used a file path in an interpolated string during a `nix repl` session.
+> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents.
+> Use `:r` to reset the repl as needed.
+
+[store path]: @docroot@/store/store-path.md
+
+Path values can be expressed as [path literals](syntax.md#path-literal).
+The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path.
+
+### Null {#type-null}
+
+There is a single value of type _null_ in the Nix language.
+
+<!-- TODO: mention the top-level environment -->
+
+This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null).
+
+## Compound values
+
+### Attribute set {#type-attrs}
+
+<!-- TODO(@rhendric, #10970): fill this out -->
+
+An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal).
+The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set.
+
+### List {#type-list}
+
+<!-- TODO(@rhendric, #10970): fill this out -->
+
+A list can be constructed with a [list literal](syntax.md#list-literal).
+The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list.
+
+## Function {#type-function}
+
+<!-- TODO(@rhendric, #10970): fill this out -->
+
+A function can be constructed with a [function expression](syntax.md#functions).
+The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function.
+
+## External {#type-external}
+
+An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files).
+Such a value can be substituted in Nix expressions but only created and used by plugin code.

doc/manual/src/language/values.md

@@ -1,280 +0,0 @@
-# Data Types
-
-## Primitives
-
-- <a id="type-string" href="#type-string">String</a>
-
-  *Strings* can be written in three ways.
-
-  The most common way is to enclose the string between double quotes,
-  e.g., `"foo bar"`. Strings can span multiple lines. The special
-  characters `"` and `\` and the character sequence `${` must be
-  escaped by prefixing them with a backslash (`\`). Newlines, carriage
-  returns and tabs can be written as `\n`, `\r` and `\t`,
-  respectively.
-
-  You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation].
-
-  [string interpolation]: ./string-interpolation.md
-
-  The second way to write string literals is as an *indented string*,
-  which is enclosed between pairs of *double single-quotes*, like so:
-
-  ```nix
-  ''
-    This is the first line.
-    This is the second line.
-      This is the third line.
-  ''
-  ```
-
-  This kind of string literal intelligently strips indentation from
-  the start of each line. To be precise, it strips from each line a
-  number of spaces equal to the minimal indentation of the string as a
-  whole (disregarding the indentation of empty lines). For instance,
-  the first and second line are indented two spaces, while the third
-  line is indented four spaces. Thus, two spaces are stripped from
-  each line, so the resulting string is
-
-  ```nix
-  "This is the first line.\nThis is the second line.\n  This is the third line.\n"
-  ```
-
-  Note that the whitespace and newline following the opening `''` is
-  ignored if there is no non-whitespace text on the initial line.
-
-  Indented strings support [string interpolation].
-
-  Since `${` and `''` have special meaning in indented strings, you
-  need a way to quote them. `$` can be escaped by prefixing it with
-  `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped
-  by prefixing it with `'`, i.e., `'''`. `$` removes any special
-  meaning from the following `$`. Linefeed, carriage-return and tab
-  characters can be written as `''\n`, `''\r`, `''\t`, and `''\`
-  escapes any other character.
-
-  Indented strings are primarily useful in that they allow multi-line
-  string literals to follow the indentation of the enclosing Nix
-  expression, and that less escaping is typically necessary for
-  strings representing languages such as shell scripts and
-  configuration files because `''` is much less common than `"`.
-  Example:
-
-  ```nix
-  stdenv.mkDerivation {
-    ...
-    postInstall =
-      ''
-        mkdir $out/bin $out/etc
-        cp foo $out/bin
-        echo "Hello World" > $out/etc/foo.conf
-        ${if enableBar then "cp bar $out/bin" else ""}
-      '';
-    ...
-  }
-  ```
-
-  Finally, as a convenience, *URIs* as defined in appendix B of
-  [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as
-  is*, without quotes. For instance, the string
-  `"http://example.org/foo.tar.bz2"` can also be written as
-  `http://example.org/foo.tar.bz2`.
-
-- <a id="type-number" href="#type-number">Number</a>
-
-  Numbers, which can be *integers* (like `123`) or *floating point*
-  (like `123.43` or `.27e13`).
-
-  See [arithmetic] and [comparison] operators for semantics.
-
-  [arithmetic]: ./operators.md#arithmetic
-  [comparison]: ./operators.md#comparison
-
-- <a id="type-path" href="#type-path">Path</a>
-
-  *Paths* are distinct from strings and can be expressed by path literals such as `./builder.sh`.
-
-  Paths are suitable for referring to local files, and are often preferable over strings.
-  - Path values do not contain trailing slashes, `.` and `..`, as they are resolved when evaluating a path literal.
-  - Path literals are automatically resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory).
-  - The files referred to by path values are automatically copied into the Nix store when used in a string interpolation or concatenation.
-  - Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file.
-
-  A path literal must contain at least one slash to be recognised as such.
-  For instance, `builder.sh` is not a path:
-  it's parsed as an expression that selects the attribute `sh` from the variable `builder`.
-
-  Path literals may also refer to absolute paths by starting with a slash.
-
-  > **Note**
-  >
-  > Absolute paths make expressions less portable.
-  > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead.
-  > This avoids some confusion about whether files at that location will be used during evaluation.
-  > It also avoids unintentional situations where some function might try to copy everything at the location into the store.
-
-  If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory.
-  For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`.
-  Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation.
-
-  Paths can be used in [string interpolation] and string concatenation.
-  For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/<hash>-foo.txt"`.
-
-  Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression.
-  For example, assume you used a file path in an interpolated string during a `nix repl` session.
-  Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. Use `:r` to reset the repl as needed.
-
-  [store path]: @docroot@/store/store-path.md
-
-  Path literals can also include [string interpolation], besides being [interpolated into other expressions].
-
-  [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions
-
-  At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.
-
-  `a.${foo}/b.${bar}` is a syntactically valid number division operation.
-  `./a.${foo}/b.${bar}` is a path.
-
-  [Lookup path](./constructs/lookup-path.md) literals such as `<nixpkgs>` also resolve to path values.
-
-- <a id="type-boolean" href="#type-boolean">Boolean</a>
-
-  *Booleans* with values `true` and `false`.
-
-- <a id="type-null" href="#type-null">Null</a>
-
-  The null value, denoted as `null`.
-
-## List
-
-Lists are formed by enclosing a whitespace-separated list of values
-between square brackets. For example,
-
-```nix
-[ 123 ./foo.nix "abc" (f { x = y; }) ]
-```
-
-defines a list of four elements, the last being the result of a call to
-the function `f`. Note that function calls have to be enclosed in
-parentheses. If they had been omitted, e.g.,
-
-```nix
-[ 123 ./foo.nix "abc" f { x = y; } ]
-```
-
-the result would be a list of five elements, the fourth one being a
-function and the fifth being a set.
-
-Note that lists are only lazy in values, and they are strict in length.
-
-Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). 
-
-## Attribute Set
-
-An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`).
-
-An attribute name can be an identifier or a [string](#string).
-An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`).
-
-> **Syntax**
->
-> *name* = *identifier* | *string* \
-> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*`
-
-Names and values are separated by an equal sign (`=`).
-Each value is an arbitrary expression terminated by a semicolon (`;`).
-
-> **Syntax**
->
-> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}`
-
-Attributes can appear in any order.
-An attribute name may only occur once.
-
-Example:
-
-```nix
-{
-  x = 123;
-  text = "Hello";
-  y = f { bla = 456; };
-}
-```
-
-This defines a set with attributes named `x`, `text`, `y`.
-
-Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection).
-
-Example:
-
-```nix
-{ a = "Foo"; b = "Bar"; }.a
-```
-
-This evaluates to `"Foo"`.
-
-It is possible to provide a default value in an attribute selection using the `or` keyword.
-
-Example:
-
-```nix
-{ a = "Foo"; b = "Bar"; }.c or "Xyzzy"
-```
-
-```nix
-{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy"
-```
-
-will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set.
-
-You can use arbitrary double-quoted strings as attribute names:
-
-```nix
-{ "$!@#?" = 123; }."$!@#?"
-```
-
-```nix
-let bar = "bar"; in
-{ "foo ${bar}" = 123; }."foo ${bar}"
-```
-
-Both will evaluate to `123`.
-
-Attribute names support [string interpolation]:
-
-```nix
-let bar = "foo"; in
-{ foo = 123; }.${bar}
-```
-
-```nix
-let bar = "foo"; in
-{ ${bar} = 123; }.foo
-```
-
-Both will evaluate to `123`.
-
-In the special case where an attribute name inside of a set declaration
-evaluates to `null` (which is normally an error, as `null` cannot be coerced to
-a string), that attribute is simply not added to the set:
-
-```nix
-{ ${if foo then "bar" else null} = true; }
-```
-
-This will evaluate to `{}` if `foo` evaluates to `false`.
-
-A set that has a `__functor` attribute whose value is callable (i.e. is
-itself a function or a set with a `__functor` attribute whose value is
-callable) can be applied as if it were a function, with the set itself
-passed in first , e.g.,
-
-```nix
-let add = { __functor = self: x: x + self.x; };
-    inc = add // { x = 1; };
-in inc 1
-```
-
-evaluates to `2`. This can be used to attach metadata to a function
-without the caller needing to treat it specially, or to implement a form
-of object-oriented programming, for example.

doc/manual/src/protocols/nix-archive.md

@@ -29,7 +29,7 @@ regular = [ str("executable"), str("") ], str("contents"), str(contents);
 symlink = str("target"), str(target);
 
 (* side condition: directory entries must be ordered by their names *)
-directory = str("type"), str("directory") { directory-entry };
+directory = { directory-entry };
 
 directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")");
 ```

doc/manual/src/protocols/tarball-fetcher.md

@@ -41,4 +41,30 @@ Link: <https://example.org/hello/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz
 For tarball flakes, the value of the `lastModified` flake attribute is
 defined as the timestamp of the newest file inside the tarball.
 
+## Gitea and Forgejo support
+
+This protocol is supported by Gitea since v1.22.1 and by Forgejo since v7.0.4/v8.0.0 and can be used with the following flake URL schema:
+
+```
+https://<domain name>/<owner>/<repo>/archive/<reference or revison>.tar.gz
+```
+
+> **Example**
+>
+>
+> ```nix
+> # flake.nix
+> {
+>    inputs = {
+>      foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz";
+>      bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz";
+>      qux = {
+>        url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz";
+>        flake = false;
+>      };
+>    };
+>    outputs = { foo, bar, qux }: { /* ... */ };
+> }
+```
+
 [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive

flake.nix

@@ -25,7 +25,6 @@
 
     let
       inherit (nixpkgs) lib;
-      inherit (lib) fileset;
 
       officialRelease = false;
 
@@ -58,6 +57,18 @@
         "stdenv"
       ];
 
+      /**
+        `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and
+        merges the results into a single attribute set.
+
+        This can be nested to form a build matrix where all the attributes
+        generated by the innermost `f` are returned as is.
+        (Provided that the names are unique.)
+
+        See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs
+       */
+      flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs;
+
       forAllSystems = lib.genAttrs systems;
 
       forAllCrossSystems = lib.genAttrs crossSystems;
@@ -117,114 +128,23 @@
         {
           nixStable = prev.nix;
 
-          default-busybox-sandbox-shell = final.busybox.override {
+          # A new scope, so that we can use `callPackage` to inject our own interdependencies
-            useMusl = true;
+          # without "polluting" the top level "`pkgs`" attrset.
-            enableStatic = true;
+          # This also has the benefit of providing us with a distinct set of packages
-            enableMinimal = true;
+          # we can iterate over.
-            extraConfig = ''
+          nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix);
-              CONFIG_FEATURE_FANCY_ECHO y
-              CONFIG_FEATURE_SH_MATH y
-              CONFIG_FEATURE_SH_MATH_64 y
 
-              CONFIG_ASH y
+          # The dependencies are in their own scope, so that they don't have to be
-              CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+          # in Nixpkgs top level `pkgs` or `nixComponents`.
-
+          nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix {
-              CONFIG_ASH_ALIAS y
+            inherit inputs stdenv versionSuffix;
-              CONFIG_ASH_BASH_COMPAT y
+            pkgs = final;
-              CONFIG_ASH_CMDCMD y
-              CONFIG_ASH_ECHO y
-              CONFIG_ASH_GETOPTS y
-              CONFIG_ASH_INTERNAL_GLOB y
-              CONFIG_ASH_JOB_CONTROL y
-              CONFIG_ASH_PRINTF y
-              CONFIG_ASH_TEST y
-            '';
-          };
-
-          libgit2-nix = final.libgit2.overrideAttrs (attrs: {
-            src = libgit2;
-            version = libgit2.lastModifiedDate;
-            cmakeFlags = attrs.cmakeFlags or []
-              ++ [ "-DUSE_SSH=exec" ];
           });
 
-          boehmgc-nix = final.boehmgc.override {
+          nix = final.nixComponents.nix;
-            enableLargeConfig = true;
-          };
 
-          libseccomp-nix = final.libseccomp.overrideAttrs (_: rec {
+          nix_noTests = final.nix.override {
-            version = "2.5.5";
+            doInstallCheck = false;
-            src = final.fetchurl {
-              url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
-              hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
-            };
-          });
-
-          nix-util = final.callPackage ./src/libutil/package.nix {
-            inherit
-              fileset
-              stdenv
-              officialRelease
-              versionSuffix
-              ;
-          };
-
-          nix-store = final.callPackage ./src/libstore/package.nix {
-            inherit
-              fileset
-              stdenv
-              officialRelease
-              versionSuffix
-              ;
-            libseccomp = final.libseccomp-nix;
-            busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
-          };
-
-          nix-fetchers = final.callPackage ./src/libfetchers/package.nix {
-            inherit
-              fileset
-              stdenv
-              officialRelease
-              versionSuffix
-              ;
-          };
-
-          nix =
-            final.callPackage ./package.nix {
-              inherit
-                fileset
-                stdenv
-                officialRelease
-                versionSuffix
-                ;
-              boehmgc = final.boehmgc-nix;
-              libgit2 = final.libgit2-nix;
-              libseccomp = final.libseccomp-nix;
-              busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
-            };
-
-          nix-perl-bindings = final.callPackage ./src/perl/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
-
-          nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
-          };
-
-          nix-external-api-docs = final.callPackage ./src/external-api-docs/package.nix {
-            inherit
-              fileset
-              stdenv
-              versionSuffix
-              ;
           };
 
           # See https://github.com/NixOS/nixpkgs/pull/214409
@@ -241,7 +161,7 @@
       # 'nix-perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
 
-      hydraJobs = import ./maintainers/hydra.nix {
+      hydraJobs = import ./packaging/hydra.nix {
         inherit
           inputs
           binaryTarball
@@ -275,31 +195,39 @@
         # the old build system is gone and we are back to one build
         # system, we should reenable this.
         #perlBindings = self.hydraJobs.perlBindings.${system};
-      } // devFlake.checks.${system} or {}
+      }
+      # Add "passthru" tests
+      // flatMapAttrs ({
+          "" = nixpkgsFor.${system}.native;
+        } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) {
+          # TODO: enable static builds for darwin, blocked on:
+          #       https://github.com/NixOS/nixpkgs/issues/320448
+          "static-" = nixpkgsFor.${system}.static;
+        })
+        (nixpkgsPrefix: nixpkgs:
+          flatMapAttrs nixpkgs.nixComponents
+            (pkgName: pkg:
+              flatMapAttrs pkg.tests or {}
+              (testName: test: {
+                "${nixpkgsPrefix}${pkgName}-${testName}" = test;
+              })
+            )
+        )
+      // devFlake.checks.${system} or {}
       );
 
-      packages = forAllSystems (system: {
+      packages = forAllSystems (system:
+        { # Here we put attributes that map 1:1 into packages.<system>, ie
+          # for which we don't apply the full build matrix such as cross or static.
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
           default = self.packages.${system}.nix;
-        nix-internal-api-docs = nixpkgsFor.${system}.native.nix-internal-api-docs;
+          nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
-        nix-external-api-docs = nixpkgsFor.${system}.native.nix-external-api-docs;
+          nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
-      } // lib.concatMapAttrs
+        }
         # We need to flatten recursive attribute sets of derivations to pass `flake check`.
-        (pkgName: {}: {
+        // flatMapAttrs
-          "${pkgName}" = nixpkgsFor.${system}.native.${pkgName};
+          { # Components we'll iterate over in the upcoming lambda
-          "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName};
-        } // lib.concatMapAttrs
-          (crossSystem: {}: {
-            "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName};
-          })
-          (lib.genAttrs crossSystems (_: { }))
-        // lib.concatMapAttrs
-          (stdenvName: {}: {
-            "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName};
-          })
-          (lib.genAttrs stdenvs (_: { })))
-        {
             "nix" = { };
             # Temporarily disabled because GitHub Actions OOM issues. Once
             # the old build system is gone and we are back to one build
@@ -308,6 +236,20 @@
             #"nix-store" = { };
             #"nix-fetchers" = { };
           }
+          (pkgName: {}: {
+              # These attributes go right into `packages.<system>`.
+              "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
+              "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName};
+            }
+            // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: {
+              # These attributes go right into `packages.<system>`.
+              "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
+            })
+            // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: {
+              # These attributes go right into `packages.<system>`.
+              "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName};
+            })
+          )
         // lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         dockerImage =
           let
@@ -336,6 +278,7 @@
             in
               "-D${prefix}:${rest}";
           havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
+          ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
         in {
           pname = "shell-for-" + attrs.pname;
 
@@ -359,28 +302,39 @@
           env = {
             # Needed for Meson to find Boost.
             # https://github.com/NixOS/nixpkgs/issues/86131.
-            BOOST_INCLUDEDIR = "${lib.getDev pkgs.boost}/include";
+            BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include";
-            BOOST_LIBRARYDIR = "${lib.getLib pkgs.boost}/lib";
+            BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib";
             # For `make format`, to work without installing pre-commit
             _NIX_PRE_COMMIT_HOOKS_CONFIG =
               "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}";
           };
 
           mesonFlags =
-            map (transformFlag "libutil") pkgs.nix-util.mesonFlags
+            map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags)
-            ++ map (transformFlag "libstore") pkgs.nix-store.mesonFlags
+            ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags)
-            ++ map (transformFlag "libfetchers") pkgs.nix-fetchers.mesonFlags
+            ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags)
-            ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nix-perl-bindings.mesonFlags)
+            ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags))
+            ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags)
+            ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags)
             ;
 
           nativeBuildInputs = attrs.nativeBuildInputs or []
-            ++ pkgs.nix-util.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-util.nativeBuildInputs
-            ++ pkgs.nix-store.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-store.nativeBuildInputs
-            ++ pkgs.nix-fetchers.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
-            ++ lib.optionals havePerl pkgs.nix-perl-bindings.nativeBuildInputs
+            ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
-            ++ pkgs.nix-internal-api-docs.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
-            ++ pkgs.nix-external-api-docs.nativeBuildInputs
+            ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
+            ++ lib.optional
+              (!stdenv.buildPlatform.canExecute stdenv.hostPlatform
+                 # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
+                 && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin)
+                 && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages
+                 && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages))
+              pkgs.buildPackages.mesonEmulatorHook
             ++ [
+              pkgs.buildPackages.cmake
+              pkgs.buildPackages.shellcheck
               modular.pre-commit.settings.package
               (pkgs.writeScriptBin "pre-commit-hooks-install"
                 modular.pre-commit.settings.installationScript)
@@ -391,6 +345,10 @@
             ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools;
 
           buildInputs = attrs.buildInputs or []
+            ++ [
+              pkgs.gtest
+              pkgs.rapidcheck
+            ]
             ++ lib.optional havePerl pkgs.perl
             ;
         });

maintainers/README.md

@@ -30,7 +30,6 @@ We aim to achieve this by improving the contributor experience and attracting mo
 ## Members
 
 - Eelco Dolstra (@edolstra) – Team lead
-- Théophane Hufschmitt (@thufschmitt)
 - Valentin Gagarin (@fricklerhandwerk)
 - Thomas Bereknyei (@tomberek)
 - Robert Hensing (@roberth)
@@ -51,7 +50,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/
        - mark it as draft if it is blocked on the contributor
        - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again.
 
-- Work meeting: [Mondays 13:00-15:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn)
+- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn)
 
   1. Code review on pull requests from [In review](#in-review).
   2. Other chores and tasks.

maintainers/flake-module.nix

@@ -18,7 +18,6 @@
             ''^tests/unit/[^/]*/data/.*$''
 
             # Don't format vendored code
-            ''^src/toml11/.*''
             ''^doc/manual/redirects\.js$''
             ''^doc/manual/theme/highlight\.js$''
 
@@ -65,14 +64,6 @@
             ''^src/libexpr/eval-settings\.hh$''
             ''^src/libexpr/eval\.cc$''
             ''^src/libexpr/eval\.hh$''
-            ''^src/libexpr/flake/config\.cc$''
-            ''^src/libexpr/flake/flake\.cc$''
-            ''^src/libexpr/flake/flake\.hh$''
-            ''^src/libexpr/flake/flakeref\.cc$''
-            ''^src/libexpr/flake/flakeref\.hh$''
-            ''^src/libexpr/flake/lockfile\.cc$''
-            ''^src/libexpr/flake/lockfile\.hh$''
-            ''^src/libexpr/flake/url-name\.cc$''
             ''^src/libexpr/function-trace\.cc$''
             ''^src/libexpr/gc-small-vector\.hh$''
             ''^src/libexpr/get-drvs\.cc$''
@@ -127,6 +118,14 @@
             ''^src/libfetchers/tarball\.hh$''
             ''^src/libfetchers/git\.cc$''
             ''^src/libfetchers/mercurial\.cc$''
+            ''^src/libflake/flake/config\.cc$''
+            ''^src/libflake/flake/flake\.cc$''
+            ''^src/libflake/flake/flake\.hh$''
+            ''^src/libflake/flake/flakeref\.cc$''
+            ''^src/libflake/flake/flakeref\.hh$''
+            ''^src/libflake/flake/lockfile\.cc$''
+            ''^src/libflake/flake/lockfile\.hh$''
+            ''^src/libflake/flake/url-name\.cc$''
             ''^src/libmain/common-args\.cc$''
             ''^src/libmain/common-args\.hh$''
             ''^src/libmain/loggers\.cc$''
@@ -144,6 +143,7 @@
             ''^src/libstore/common-protocol-impl\.hh$''
             ''^src/libstore/common-protocol\.cc$''
             ''^src/libstore/common-protocol\.hh$''
+            ''^src/libstore/common-ssh-store-config\.hh$''
             ''^src/libstore/content-address\.cc$''
             ''^src/libstore/content-address\.hh$''
             ''^src/libstore/daemon\.cc$''
@@ -216,7 +216,6 @@
             ''^src/libstore/serve-protocol\.hh$''
             ''^src/libstore/sqlite\.cc$''
             ''^src/libstore/sqlite\.hh$''
-            ''^src/libstore/ssh-store-config\.hh$''
             ''^src/libstore/ssh-store\.cc$''
             ''^src/libstore/ssh\.cc$''
             ''^src/libstore/ssh\.hh$''
@@ -429,14 +428,13 @@
             ''^tests/functional/test-libstoreconsumer/main\.cc''
             ''^tests/nixos/ca-fd-leak/sender\.c''
             ''^tests/nixos/ca-fd-leak/smuggler\.c''
+            ''^tests/nixos/user-sandboxing/attacker\.c''
             ''^tests/unit/libexpr-support/tests/libexpr\.hh''
             ''^tests/unit/libexpr-support/tests/value/context\.cc''
             ''^tests/unit/libexpr-support/tests/value/context\.hh''
             ''^tests/unit/libexpr/derived-path\.cc''
             ''^tests/unit/libexpr/error_traces\.cc''
             ''^tests/unit/libexpr/eval\.cc''
-            ''^tests/unit/libexpr/flake/flakeref\.cc''
-            ''^tests/unit/libexpr/flake/url-name\.cc''
             ''^tests/unit/libexpr/json\.cc''
             ''^tests/unit/libexpr/main\.cc''
             ''^tests/unit/libexpr/primops\.cc''
@@ -445,9 +443,10 @@
             ''^tests/unit/libexpr/value/context\.cc''
             ''^tests/unit/libexpr/value/print\.cc''
             ''^tests/unit/libfetchers/public-key\.cc''
+            ''^tests/unit/libflake/flakeref\.cc''
+            ''^tests/unit/libflake/url-name\.cc''
             ''^tests/unit/libstore-support/tests/derived-path\.cc''
             ''^tests/unit/libstore-support/tests/derived-path\.hh''
-            ''^tests/unit/libstore-support/tests/libstore\.hh''
             ''^tests/unit/libstore-support/tests/nix_api_store\.hh''
             ''^tests/unit/libstore-support/tests/outputs-spec\.cc''
             ''^tests/unit/libstore-support/tests/outputs-spec\.hh''
@@ -496,11 +495,9 @@
           excludes = [
             # We haven't linted these files yet
             ''^config/install-sh$''
-            ''^misc/systemv/nix-daemon$''
             ''^misc/bash/completion\.sh$''
             ''^misc/fish/completion\.fish$''
             ''^misc/zsh/completion\.zsh$''
-            ''^scripts/check-hydra-status\.sh$''
             ''^scripts/create-darwin-volume\.sh$''
             ''^scripts/install-darwin-multi-user\.sh$''
             ''^scripts/install-multi-user\.sh$''
@@ -522,6 +519,7 @@
             ''^tests/functional/ca/repl\.sh$''
             ''^tests/functional/ca/selfref-gc\.sh$''
             ''^tests/functional/ca/why-depends\.sh$''
+            ''^tests/functional/characterisation-test-infra\.sh$''
             ''^tests/functional/check\.sh$''
             ''^tests/functional/common/vars-and-functions\.sh$''
             ''^tests/functional/completions\.sh$''
@@ -579,9 +577,7 @@
             ''^tests/functional/impure-env\.sh$''
             ''^tests/functional/impure-eval\.sh$''
             ''^tests/functional/install-darwin\.sh$''
-            ''^tests/functional/lang-test-infra\.sh$''
             ''^tests/functional/lang\.sh$''
-            ''^tests/functional/lang/framework\.sh$''
             ''^tests/functional/legacy-ssh-store\.sh$''
             ''^tests/functional/linux-sandbox\.sh$''
             ''^tests/functional/local-overlay-store/add-lower-inner\.sh$''

meson.build

@@ -6,9 +6,39 @@ project('nix-dev-shell', 'cpp',
   subproject_dir : 'src',
 )
 
+# Internal Libraries
 subproject('libutil')
 subproject('libstore')
 subproject('libfetchers')
-subproject('perl')
+subproject('libexpr')
+subproject('libflake')
+subproject('libmain')
+subproject('libcmd')
+
+# Executables
+subproject('nix')
+
+# Docs
 subproject('internal-api-docs')
 subproject('external-api-docs')
+
+# External C wrapper libraries
+subproject('libutil-c')
+subproject('libstore-c')
+subproject('libexpr-c')
+subproject('libmain-c')
+
+# Language Bindings
+if not meson.is_cross_build()
+  subproject('perl')
+endif
+
+# Testing
+subproject('nix-util-test-support')
+subproject('nix-util-tests')
+subproject('nix-store-test-support')
+subproject('nix-store-tests')
+subproject('nix-fetchers-tests')
+subproject('nix-expr-test-support')
+subproject('nix-expr-tests')
+subproject('nix-flake-tests')

misc/systemv/nix-daemon

@@ -34,6 +34,7 @@ else
 fi
 
 # Source function library.
+# shellcheck source=/dev/null
 . /etc/init.d/functions
 
 LOCKFILE=/var/lock/subsys/nix-daemon
@@ -41,14 +42,20 @@ RUNDIR=/var/run/nix
 PIDFILE=${RUNDIR}/nix-daemon.pid
 RETVAL=0
 
-base=${0##*/}
+# https://www.shellcheck.net/wiki/SC3004
+# Check if gettext exists
+if ! type gettext > /dev/null 2>&1
+then
+  # If not, create a dummy function that returns the input verbatim
+  gettext() { printf '%s' "$1"; }
+fi
 
 start() {
 
     mkdir -p ${RUNDIR}
     chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR}
 
-    echo -n $"Starting nix daemon... "
+    printf '%s' "$(gettext 'Starting nix daemon... ')"
 
     daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS
     RETVAL=$?
@@ -58,7 +65,7 @@ start() {
 }
 
 stop() {
-    echo -n $"Shutting down nix daemon: "
+    printf '%s' "$(gettext 'Shutting down nix daemon: ')"
     killproc -p ${PIDFILE} $NIX_DAEMON_BIN
     RETVAL=$?
     [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE}
@@ -67,7 +74,7 @@ stop() {
 }
 
 reload() {
-    echo -n $"Reloading nix daemon... "
+    printf '%s' "$(gettext 'Reloading nix daemon... ')"
     killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP
     RETVAL=$?
     echo
@@ -105,7 +112,7 @@ case "$1" in
         fi
     ;;
   *)
-        echo $"Usage: $0 {start|stop|status|restart|condrestart}"
+        printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")"
         exit 2
     ;;
 esac

mk/common-test.sh

@@ -4,7 +4,7 @@
 # remove file extension.
 
 test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \
-    -e "s|^tests/unit/[^/]*/data/||" \
+    -e "s|^src/[^/]*-test/data/||" \
     -e "s|^tests/functional/||" \
     -e "s|\.sh$||" \
     )

package.nix

@@ -1,12 +1,10 @@
 { lib
-, fetchurl
 , stdenv
 , releaseTools
 , autoconf-archive
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
-, buildPackages
 , nlohmann_json
 , bison
 , boost
@@ -15,7 +13,6 @@
 , curl
 , editline
 , readline
-, fileset
 , flex
 , git
 , gtest
@@ -35,7 +32,8 @@
 , pkg-config
 , rapidcheck
 , sqlite
-, util-linux
+, toml11
+, unixtools
 , xz
 
 , busybox-sandbox-shell ? null
@@ -50,7 +48,6 @@
 , pname ? "nix"
 
 , versionSuffix ? ""
-, officialRelease ? false
 
 # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix
 , doBuild ? true
@@ -113,6 +110,8 @@
 }:
 
 let
+  inherit (lib) fileset;
+
   version = lib.fileContents ./.version + versionSuffix;
 
   # selected attributes with defaults, will be used to define some
@@ -176,7 +175,7 @@ in {
           (fileset.difference ./src ./src/perl)
           ./COPYING
           ./scripts/local.mk
-        ] ++ lib.optionals buildUnitTests [
+        ] ++ lib.optionals enableManual [
           ./doc/manual
         ] ++ lib.optionals buildUnitTests [
           ./tests/unit
@@ -214,11 +213,10 @@ in {
     man # for testing `nix-* --help`
   ] ++ lib.optionals (doInstallCheck || enableManual) [
     jq # Also for custom mdBook preprocessor.
-  ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
+  ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump
   ;
 
   buildInputs = lib.optionals doBuild [
-    boost
     brotli
     bzip2
     curl
@@ -227,6 +225,7 @@ in {
     libsodium
     openssl
     sqlite
+    toml11
     xz
     ({ inherit readline editline; }.${readlineFlavor})
   ] ++ lib.optionals enableMarkdown [
@@ -245,34 +244,13 @@ in {
   ;
 
   propagatedBuildInputs = [
+    boost
     nlohmann_json
   ] ++ lib.optional enableGC boehmgc;
 
   dontBuild = !attrs.doBuild;
   doCheck = attrs.doCheck;
 
-  disallowedReferences = [ boost ];
-
-  preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) (
-    ''
-      # Copy libboost_context so we don't get all of Boost in our closure.
-      # https://github.com/NixOS/nixpkgs/issues/45462
-      mkdir -p $out/lib
-      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
-      rm -f $out/lib/*.a
-    '' + lib.optionalString stdenv.hostPlatform.isLinux ''
-      chmod u+w $out/lib/*.so.*
-      patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-    '' + lib.optionalString stdenv.hostPlatform.isDarwin ''
-      for LIB in $out/lib/*.dylib; do
-        chmod u+w $LIB
-        install_name_tool -id $LIB $LIB
-        install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
-      done
-      install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
-    ''
-  );
-
   configureFlags = [
     (lib.enableFeature doBuild "build")
     (lib.enableFeature buildUnitTests "unit-tests")
@@ -319,11 +297,6 @@ in {
     lib.optionalString stdenv.hostPlatform.isStatic ''
       mkdir -p $out/nix-support
       echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
-    '' + lib.optionalString stdenv.isDarwin ''
-      install_name_tool \
-      -change ${boost}/lib/libboost_context.dylib \
-      $out/lib/libboost_context.dylib \
-      $out/lib/libnixutil.dylib
     ''
   ) + lib.optionalString enableManual ''
     mkdir -p ''${!outputDoc}/nix-support

packaging/components.nix

@@ -0,0 +1,43 @@
+scope:
+let
+  inherit (scope) callPackage;
+in
+
+# This becomes the pkgs.nixComponents attribute set
+{
+  nix = callPackage ../package.nix { };
+
+  nix-util = callPackage ../src/libutil/package.nix { };
+  nix-util-c = callPackage ../src/libutil-c/package.nix { };
+  nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { };
+  nix-util-tests = callPackage ../tests/unit/libutil/package.nix { };
+
+  nix-store = callPackage ../src/libstore/package.nix { };
+  nix-store-c = callPackage ../src/libstore-c/package.nix { };
+  nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { };
+  nix-store-tests = callPackage ../tests/unit/libstore/package.nix { };
+
+  nix-fetchers = callPackage ../src/libfetchers/package.nix { };
+  nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { };
+
+  nix-expr = callPackage ../src/libexpr/package.nix { };
+  nix-expr-c = callPackage ../src/libexpr-c/package.nix { };
+  nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { };
+  nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { };
+
+  nix-flake = callPackage ../src/libflake/package.nix { };
+  nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { };
+
+  nix-main = callPackage ../src/libmain/package.nix { };
+  nix-main-c = callPackage ../src/libmain-c/package.nix { };
+
+  nix-cmd = callPackage ../src/libcmd/package.nix { };
+
+  # Will replace `nix` once the old build system is gone.
+  nix-ng = callPackage ../src/nix/package.nix { };
+
+  nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { };
+  nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { };
+
+  nix-perl-bindings = callPackage ../src/perl/package.nix { };
+}

packaging/dependencies.nix

@@ -0,0 +1,134 @@
+# These overrides are applied to the dependencies of the Nix components.
+
+{
+  # Flake inputs; used for sources
+  inputs,
+
+  # The raw Nixpkgs, not affected by this scope
+  pkgs,
+
+  stdenv,
+  versionSuffix,
+}:
+
+let
+  prevStdenv = stdenv;
+in
+
+let
+  inherit (pkgs) lib;
+
+  root = ../.;
+
+  stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64
+    then darwinStdenv
+    else prevStdenv;
+
+  # Fix the following error with the default x86_64-darwin SDK:
+  #
+  #     error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer
+  #
+  # Despite the use of the 10.13 deployment target here, the aligned
+  # allocation function Clang uses with this setting actually works
+  # all the way back to 10.6.
+  darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; };
+
+  # Nixpkgs implements this by returning a subpath into the fetched Nix sources.
+  resolvePath = p: p;
+
+  # Indirection for Nixpkgs to override when package.nix files are vendored
+  filesetToSource = lib.fileset.toSource;
+
+  localSourceLayer = finalAttrs: prevAttrs:
+    let
+      workDirPath =
+        # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has
+        # the requirement that everything except passthru and meta must be
+        # serialized by mkDerivation, which doesn't work for this.
+        prevAttrs.workDir;
+
+      workDirSubpath = lib.path.removePrefix root workDirPath;
+      sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset;
+      src = lib.fileset.toSource { fileset = sources; inherit root; };
+
+    in
+    {
+      sourceRoot = "${src.name}/" + workDirSubpath;
+      inherit src;
+
+      # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir.
+      fileset = null;
+      workDir = null;
+    };
+
+in
+scope: {
+  inherit stdenv versionSuffix;
+  version = lib.fileContents ../.version + versionSuffix;
+
+  libseccomp = pkgs.libseccomp.overrideAttrs (_: rec {
+    version = "2.5.5";
+    src = pkgs.fetchurl {
+      url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
+      hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
+    };
+  });
+
+  boehmgc = pkgs.boehmgc.override {
+    enableLargeConfig = true;
+  };
+
+  # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
+  boost = (pkgs.boost.override {
+    extraB2Args = [
+      "--with-container"
+      "--with-context"
+      "--with-coroutine"
+    ];
+  }).overrideAttrs (old: {
+    # Need to remove `--with-*` to use `--with-libraries=...`
+    buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase;
+    installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase;
+  });
+
+  libgit2 = pkgs.libgit2.overrideAttrs (attrs: {
+    src = inputs.libgit2;
+    version = inputs.libgit2.lastModifiedDate;
+    cmakeFlags = attrs.cmakeFlags or []
+      ++ [ "-DUSE_SSH=exec" ];
+  });
+
+  busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
+    useMusl = true;
+    enableStatic = true;
+    enableMinimal = true;
+    extraConfig = ''
+      CONFIG_FEATURE_FANCY_ECHO y
+      CONFIG_FEATURE_SH_MATH y
+      CONFIG_FEATURE_SH_MATH_64 y
+
+      CONFIG_ASH y
+      CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+      CONFIG_ASH_ALIAS y
+      CONFIG_ASH_BASH_COMPAT y
+      CONFIG_ASH_CMDCMD y
+      CONFIG_ASH_ECHO y
+      CONFIG_ASH_GETOPTS y
+      CONFIG_ASH_INTERNAL_GLOB y
+      CONFIG_ASH_JOB_CONTROL y
+      CONFIG_ASH_PRINTF y
+      CONFIG_ASH_TEST y
+    '';
+  });
+
+  # TODO change in Nixpkgs, Windows works fine. First commit of
+  # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix.
+  toml11 = pkgs.toml11.overrideAttrs (old: {
+    meta.platforms = lib.platforms.all;
+  });
+
+  inherit resolvePath filesetToSource;
+
+  mkMesonDerivation = f: stdenv.mkDerivation (lib.extends localSourceLayer f);
+}

packaging/hydra.nix

@@ -9,7 +9,6 @@
 }:
 let
   inherit (inputs) nixpkgs nixpkgs-regression;
-  inherit (lib) fileset;
 
   installScriptFor = tarballs:
     nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix {
@@ -25,46 +24,58 @@ let
            lib.versionAtLeast client.version "2.4pre20211005")
           "-${client.version}-against-${daemon.version}";
 
-      inherit fileset;
-
       test-client = client;
       test-daemon = daemon;
 
       doBuild = false;
     };
 
+  # Technically we could just return `pkgs.nixComponents`, but for Hydra it's
+  # convention to transpose it, and to transpose it efficiently, we need to
+  # enumerate them manually, so that we don't evaluate unnecessary package sets.
   forAllPackages = lib.genAttrs [
     "nix"
     "nix-util"
+    "nix-util-c"
+    "nix-util-test-support"
+    "nix-util-tests"
     "nix-store"
+    "nix-store-c"
+    "nix-store-test-support"
+    "nix-store-tests"
     "nix-fetchers"
+    "nix-fetchers-tests"
+    "nix-expr"
+    "nix-expr-c"
+    "nix-expr-test-support"
+    "nix-expr-tests"
+    "nix-flake"
+    "nix-flake-tests"
+    "nix-main"
+    "nix-main-c"
+    "nix-cmd"
+    "nix-ng"
   ];
 in
 {
   # Binary package for various platforms.
   build = forAllPackages (pkgName:
-    forAllSystems (system: nixpkgsFor.${system}.native.${pkgName}));
+    forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName}));
 
   shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
 
   buildStatic = forAllPackages (pkgName:
-    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName}));
+    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName}));
 
   buildCross = forAllPackages (pkgName:
     forAllCrossSystems (crossSystem:
-      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName})));
+      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})));
 
   buildNoGc = forAllSystems (system:
     self.packages.${system}.nix.override { enableGC = false; }
   );
 
-  buildNoTests = forAllSystems (system:
+  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests);
-    self.packages.${system}.nix.override {
-      doCheck = false;
-      doInstallCheck = false;
-      installUnitTests = false;
-    }
-  );
 
   # Toggles some settings for better coverage. Windows needs these
   # library combinations, and Debian build Nix with GNU readline too.
@@ -76,7 +87,7 @@ in
   );
 
   # Perl bindings for various platforms.
-  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix-perl-bindings);
+  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings);
 
   # Binary tarball for various platforms, containing a Nix store
   # with the closure of 'nix' package, and the second half of
@@ -108,7 +119,7 @@ in
   installerScriptForGHA = installScriptFor [
     # Native
     self.hydraJobs.binaryTarball."x86_64-linux"
-    self.hydraJobs.binaryTarball."x86_64-darwin"
+    self.hydraJobs.binaryTarball."aarch64-darwin"
     # Cross
     self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
     self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
@@ -125,10 +136,10 @@ in
   };
 
   # API docs for Nix's unstable internal C++ interfaces.
-  internal-api-docs = nixpkgsFor.x86_64-linux.native.nix-internal-api-docs;
+  internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs;
 
   # API docs for Nix's C bindings.
-  external-api-docs = nixpkgsFor.x86_64-linux.native.nix-external-api-docs;
+  external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs;
 
   # System tests.
   tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // {

scripts/check-hydra-status.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-# set -x
-
-
-# mapfile BUILDS_FOR_LATEST_EVAL < <(
-# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
-#   jq -r '.evals[0].builds[] | @sh')
-BUILDS_FOR_LATEST_EVAL=$(
-curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
-  jq -r '.evals[0].builds[]')
-
-someBuildFailed=0
-
-for buildId in $BUILDS_FOR_LATEST_EVAL; do
-  buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
-
-  finished=$(echo "$buildInfo" | jq -r '.finished')
-
-  if [[ $finished = 0 ]]; then
-    continue
-  fi
-
-  buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus')
-
-  if [[ $buildStatus != 0 ]]; then
-    someBuildFailed=1
-    echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra: $buildInfo"
-  fi
-done
-
-exit "$someBuildFailed"

src/build-remote/build-remote.cc

@@ -11,11 +11,13 @@
 
 #include "machines.hh"
 #include "shared.hh"
+#include "plugin.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "serialise.hh"
 #include "build-result.hh"
 #include "store-api.hh"
+#include "strings.hh"
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"

src/external-api-docs/package.nix

@@ -1,7 +1,5 @@
 { lib
-, stdenv
+, mkMesonDerivation
-, releaseTools
-, fileset
 
 , meson
 , ninja
@@ -9,20 +7,25 @@
 
 # Configuration Options
 
-, versionSuffix ? ""
+, version
 }:
 
-stdenv.mkDerivation (finalAttrs: {
+let
-  pname = "nix-external-api-docs";
+  inherit (lib) fileset;
-  version = lib.fileContents ./.version + versionSuffix;
+in
 
-  src = fileset.toSource {
+mkMesonDerivation (finalAttrs: {
-    root = ../..;
+  pname = "nix-external-api-docs";
+  inherit version;
+
+  workDir = ./.;
   fileset =
     let
       cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h");
     in
     fileset.unions [
+      ./.version
+      ../../.version
       ./meson.build
       ./doxygen.cfg.in
       ./README.md
@@ -32,7 +35,6 @@ stdenv.mkDerivation (finalAttrs: {
       (cpp ../libstore-c)
       (cpp ../libutil-c)
     ];
-  };
 
   nativeBuildInputs = [
     meson
@@ -40,14 +42,10 @@ stdenv.mkDerivation (finalAttrs: {
     doxygen
   ];
 
-  postUnpack = ''
-    sourceRoot=$sourceRoot/src/external-api-docs
-  '';
-
   preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix
     ''
-      echo ${finalAttrs.version} > .version
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
     '';
 
   postInstall = ''

src/internal-api-docs/doxygen.cfg.in

@@ -38,27 +38,27 @@ GENERATE_LATEX         = NO
 # so they can expand variables despite configure variables.
 
 INPUT                  = \
-  @src@/src/libcmd \
+  @src@/libcmd \
-  @src@/src/libexpr \
+  @src@/libexpr \
-  @src@/src/libexpr/flake \
+  @src@/libexpr/flake \
-  @src@/tests/unit/libexpr \
+  @src@/nix-expr-tests \
-  @src@/tests/unit/libexpr/value \
+  @src@/nix-expr-tests/value \
-  @src@/tests/unit/libexpr/test \
+  @src@/nix-expr-test-support/test \
-  @src@/tests/unit/libexpr/test/value \
+  @src@/nix-expr-test-support/test/value \
-  @src@/src/libexpr/value \
+  @src@/libexpr/value \
-  @src@/src/libfetchers \
+  @src@/libfetchers \
-  @src@/src/libmain \
+  @src@/libmain \
-  @src@/src/libstore \
+  @src@/libstore \
-  @src@/src/libstore/build \
+  @src@/libstore/build \
-  @src@/src/libstore/builtins \
+  @src@/libstore/builtins \
-  @src@/tests/unit/libstore \
+  @src@/nix-store-tests \
-  @src@/tests/unit/libstore/test \
+  @src@/nix-store-test-support/test \
-  @src@/src/libutil \
+  @src@/libutil \
-  @src@/tests/unit/libutil \
+  @src@/nix-util-tests \
-  @src@/tests/unit/libutil/test \
+  @src@/nix-util-test-support/test \
-  @src@/src/nix \
+  @src@/nix \
-  @src@/src/nix-env \
+  @src@/nix-env \
-  @src@/src/nix-store
+  @src@/nix-store
 
 # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
 # in the source code. If set to NO, only conditional compilation will be

src/internal-api-docs/meson.build

@@ -12,7 +12,7 @@ doxygen_cfg = configure_file(
   configuration : {
     'PROJECT_NUMBER': meson.project_version(),
     'OUTPUT_DIRECTORY' : meson.current_build_dir(),
-    'src' : fs.parent(fs.parent(meson.project_source_root())),
+    'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src',
   },
 )
 

src/internal-api-docs/package.nix

@@ -1,7 +1,5 @@
 { lib
-, stdenv
+, mkMesonDerivation
-, releaseTools
-, fileset
 
 , meson
 , ninja
@@ -9,26 +7,29 @@
 
 # Configuration Options
 
-, versionSuffix ? ""
+, version
 }:
 
-stdenv.mkDerivation (finalAttrs: {
+let
-  pname = "nix-internal-api-docs";
+  inherit (lib) fileset;
-  version = lib.fileContents ./.version + versionSuffix;
+in
 
-  src = fileset.toSource {
+mkMesonDerivation (finalAttrs: {
-    root = ../..;
+  pname = "nix-internal-api-docs";
+  inherit version;
+
+  workDir = ./.;
   fileset = let
     cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh");
   in fileset.unions [
+    ./.version
+    ../../.version
     ./meson.build
     ./doxygen.cfg.in
     # Source is not compiled, but still must be available for Doxygen
     # to gather comments.
     (cpp ../.)
-      (cpp ../../tests/unit)
   ];
-  };
 
   nativeBuildInputs = [
     meson
@@ -36,14 +37,10 @@ stdenv.mkDerivation (finalAttrs: {
     doxygen
   ];
 
-  postUnpack = ''
-    sourceRoot=$sourceRoot/src/internal-api-docs
-  '';
-
   preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix
     ''
-      echo ${finalAttrs.version} > .version
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
     '';
 
   postInstall = ''

src/libcmd/.version

@@ -0,0 +1 @@
+../../.version

src/libcmd/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libcmd/built-path.cc

@@ -1,6 +1,7 @@
 #include "built-path.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "comparator.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -8,30 +9,24 @@
 
 namespace nix {
 
-#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
+// Custom implementation to avoid `ref` ptr equality
-    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
+GENERATE_CMP_EXT(
-    { \
+    ,
-        const MY_TYPE* me = this; \
+    std::strong_ordering,
-        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
+    SingleBuiltPathBuilt,
-        me = &other; \
+    *me->drvPath,
-        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
+    me->output);
-        return fields1 COMPARATOR fields2; \
-    }
-#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
 
-#define FIELD_TYPE std::pair<std::string, StorePath>
+// Custom implementation to avoid `ref` ptr equality
-CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
-#undef FIELD_TYPE
 
-#define FIELD_TYPE std::map<std::string, StorePath>
+// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on
-CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
+// Darwin, per header.
-#undef FIELD_TYPE
+GENERATE_EQUAL(
-
+    ,
-#undef CMP
+    BuiltPathBuilt ::,
-#undef CMP_ONE
+    BuiltPathBuilt,
+    *me->drvPath,
+    me->outputs);
 
 StorePath SingleBuiltPath::outPath() const
 {

src/libcmd/built-path.hh

@@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt {
     static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
 
-    DECLARE_CMP(SingleBuiltPathBuilt);
+    bool operator ==(const SingleBuiltPathBuilt &) const noexcept;
+    std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept;
 };
 
 using _SingleBuiltPathRaw = std::variant<
@@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = SingleBuiltPathBuilt;
 
+    bool operator == (const SingleBuiltPath &) const = default;
+    auto operator <=> (const SingleBuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }
@@ -59,11 +63,13 @@ struct BuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
+    bool operator == (const BuiltPathBuilt &) const noexcept;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept;
+
     std::string to_string(const StoreDirConfig & store) const;
     static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
-
-    DECLARE_CMP(BuiltPathBuilt);
 };
 
 using _BuiltPathRaw = std::variant<
@@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = BuiltPathBuilt;
 
+    bool operator == (const BuiltPath &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=> (const BuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }

src/libcmd/command.cc

@@ -1,3 +1,5 @@
+#include <nlohmann/json.hpp>
+
 #include "command.hh"
 #include "markdown.hh"
 #include "store-api.hh"
@@ -6,8 +8,7 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 #include "repl.hh"
-
+#include "strings.hh"
-#include <nlohmann/json.hpp>
 
 extern char * * environ __attribute__((weak));
 
@@ -127,12 +128,12 @@ ref<EvalState> EvalCommand::getEvalState()
     if (!evalState) {
         evalState =
             #if HAVE_BOEHMGC
-            std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
+            std::allocate_shared<EvalState>(
-                lookupPath, getEvalStore(), getStore())
+                traceable_allocator<EvalState>(),
             #else
             std::make_shared<EvalState>(
-                lookupPath, getEvalStore(), getStore())
             #endif
+                lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore())
             ;
 
         evalState->repair = repair;

src/libcmd/common-eval-args.cc

@@ -1,18 +1,57 @@
+#include "fetch-settings.hh"
 #include "eval-settings.hh"
 #include "common-eval-args.hh"
 #include "shared.hh"
+#include "config-global.hh"
 #include "filetransfer.hh"
 #include "eval.hh"
 #include "fetchers.hh"
 #include "registry.hh"
 #include "flake/flakeref.hh"
+#include "flake/settings.hh"
 #include "store-api.hh"
 #include "command.hh"
 #include "tarball.hh"
 #include "fetch-to-store.hh"
+#include "compatibility-settings.hh"
+#include "eval-settings.hh"
 
 namespace nix {
 
+fetchers::Settings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
+EvalSettings evalSettings {
+    settings.readOnlyMode,
+    {
+        {
+            "flake",
+            [](ref<Store> store, std::string_view rest) {
+                experimentalFeatureSettings.require(Xp::Flakes);
+                // FIXME `parseFlakeRef` should take a `std::string_view`.
+                auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
+                debug("fetching flake search path element '%s''", rest);
+                auto storePath = flakeRef.resolve(store).fetchTree(store).first;
+                return store->toRealPath(storePath);
+            },
+        },
+    },
+};
+
+static GlobalConfig::Register rEvalSettings(&evalSettings);
+
+
+flake::Settings flakeSettings;
+
+static GlobalConfig::Register rFlakeSettings(&flakeSettings);
+
+
+CompatibilitySettings compatibilitySettings {};
+
+static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings);
+
+
 MixEvalArgs::MixEvalArgs()
 {
     addFlag({
@@ -54,7 +93,7 @@ MixEvalArgs::MixEvalArgs()
         .description = R"(
   Add *path* to the Nix search path. The Nix search path is
   initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment
-  variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle
+  variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle
   brackets (i.e., `<nixpkgs>`).
 
   For instance, passing
@@ -144,8 +183,8 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
-            auto from = parseFlakeRef(_from, absPath("."));
+            auto from = parseFlakeRef(fetchSettings, _from, absPath("."));
-            auto to = parseFlakeRef(_to, absPath("."));
+            auto to = parseFlakeRef(fetchSettings, _to, absPath("."));
             fetchers::Attrs extraAttrs;
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
@@ -175,7 +214,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
         auto v = state.allocValue();
         std::visit(overloaded {
             [&](const AutoArgExpr & arg) {
-                state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath(".")));
+                state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath(".")));
             },
             [&](const AutoArgString & arg) {
                 v->mkString(arg.s);
@@ -203,7 +242,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
-        auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
+        auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
         auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }

src/libcmd/common-eval-args.hh

@@ -11,10 +11,37 @@
 namespace nix {
 
 class Store;
+
+namespace fetchers { struct Settings; }
+
 class EvalState;
+struct EvalSettings;
+struct CompatibilitySettings;
 class Bindings;
 struct SourcePath;
 
+namespace flake { struct Settings; }
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern fetchers::Settings fetchSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern EvalSettings evalSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern flake::Settings flakeSettings;
+
+/**
+ * Settings that control behaviors that have changed since Nix 2.3.
+ */
+extern CompatibilitySettings compatibilitySettings;
+
 struct MixEvalArgs : virtual Args, virtual MixRepair
 {
     static constexpr auto category = "Common evaluation options";

src/libcmd/compatibility-settings.hh

@@ -0,0 +1,36 @@
+#pragma once
+#include "config.hh"
+
+namespace nix {
+struct CompatibilitySettings : public Config
+{
+
+    CompatibilitySettings() = default;
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"(
+        Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified.
+
+        Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellShebangArgumentsRelativeToScript{
+        this, true, "nix-shell-shebang-arguments-relative-to-script", R"(
+        Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory.
+
+        Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+};
+
+};

src/libcmd/installable-flake.cc

@@ -43,20 +43,6 @@ std::vector<std::string> InstallableFlake::getActualAttrPaths()
     return res;
 }
 
-Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
-{
-    auto vFlake = state.allocValue();
-
-    callFlake(state, lockedFlake, *vFlake);
-
-    auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
-    assert(aOutputs);
-
-    state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
-
-    return aOutputs->value;
-}
-
 static std::string showAttrPaths(const std::vector<std::string> & paths)
 {
     std::string s;
@@ -210,7 +196,8 @@ std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
         flake::LockFlags lockFlagsApplyConfig = lockFlags;
         // FIXME why this side effect?
         lockFlagsApplyConfig.applyNixConfig = true;
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(
+            flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }

src/libcmd/installable-flake.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "common-eval-args.hh"
 #include "installable-value.hh"
 
 namespace nix {
@@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue
 
     std::vector<std::string> getActualAttrPaths();
 
-    Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
-
     DerivedPathsWithInfo toDerivedPaths() override;
 
     std::pair<Value *, PosIdx> toValue(EvalState & state) override;
@@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue
  */
 static inline FlakeRef defaultNixpkgsFlakeRef()
 {
-    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+    return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}});
 }
 
 ref<eval_cache::EvalCache> openEvalCache(

src/libcmd/installable-value.hh

@@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo
 };
 
 /**
- * An Installable which corresponds a Nix langauge value, in addition to
+ * An Installable which corresponds a Nix language value, in addition to
  * a collection of \ref DerivedPath "derived paths".
  */
 struct InstallableValue : Installable

src/libcmd/installables.cc

@@ -27,6 +27,8 @@
 
 #include <nlohmann/json.hpp>
 
+#include "strings-inline.hh"
+
 namespace nix {
 
 void completeFlakeInputPath(
@@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true));
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
@@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&](std::string flakeRef) {
             auto evalState = getEvalState();
             auto flake = flake::lockFlake(
+                flakeSettings,
                 *evalState,
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir())),
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())),
                 { .writeLockFile = false });
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
                 if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
                     overrideRegistry(
-                        fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}),
+                        fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
                         {});
                 }
@@ -289,10 +292,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             if (v2.type() == nAttrs) {
                 for (auto & i : *v2.attrs()) {
-                    std::string name = state->symbols[i.name];
+                    std::string_view name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
-                            completions.add(name);
+                            completions.add(std::string(name));
                         else
                             completions.add(prefix_ + "." + name);
                     }
@@ -338,10 +341,11 @@ void completeFlakeRefWithFragment(
             auto flakeRefS = std::string(prefix.substr(0, hash));
 
             // TODO: ideally this would use the command base directory instead of assuming ".".
-            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
+            auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
-                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
+                std::make_shared<flake::LockedFlake>(lockFlake(
+                    flakeSettings, *evalState, flakeRef, lockFlags)));
 
             auto root = evalCache->getRoot();
 
@@ -372,6 +376,7 @@ void completeFlakeRefWithFragment(
                         auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
+                        // FIXME: handle names with dots
                         completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
                     }
                 }
@@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
     Args::completeDir(completions, 0, prefix);
 
     /* Look for registry entries that match the prefix. */
-    for (auto & registry : fetchers::getRegistries(store)) {
+    for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
         for (auto & entry : registry->entries) {
             auto from = entry.from.to_string();
             if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
@@ -534,7 +539,8 @@ Installables SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir()));
+                auto [flakeRef, fragment] = parseFlakeRefWithFragment(
+                    fetchSettings, std::string { prefix }, absPath(getCommandBaseDir()));
                 result.push_back(make_ref<InstallableFlake>(
                         this,
                         getEvalState(),
@@ -851,6 +857,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     std::vector<FlakeRef> res;
     for (auto i : rawInstallables)
         res.push_back(parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(i),
             absPath(getCommandBaseDir())).first);
     return res;
@@ -873,6 +880,7 @@ std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
 {
     return {
         parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(_installable),
             absPath(getCommandBaseDir())).first
     };

src/libcmd/local.mk

@@ -6,10 +6,10 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain)
+libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain)
 
 libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 
-libcmd_LIBS = libstore libutil libexpr libmain libfetchers
+libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain
 
 $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/meson.build

@@ -0,0 +1,128 @@
+project('nix-cmd', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+  dependency('nix-expr'),
+  dependency('nix-flake'),
+  dependency('nix-main'),
+]
+subdir('build-utils-meson/subprojects')
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown'))
+deps_private += lowdown
+configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
+
+readline_flavor = get_option('readline-flavor')
+if readline_flavor == 'editline'
+  editline = dependency('libeditline', 'editline', version : '>=1.14')
+  deps_private += editline
+elif readline_flavor == 'readline'
+  readline = dependency('readline')
+  deps_private += readline
+  configdata.set(
+    'USE_READLINE',
+    1,
+    description: 'Use readline instead of editline',
+  )
+else
+  error('illegal editline flavor', readline_flavor)
+endif
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-cmd.hh',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  '-include', 'config-expr.hh',
+  '-include', 'config-main.hh',
+  '-include', 'config-cmd.hh',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'built-path.cc',
+  'command-installable-value.cc',
+  'command.cc',
+  'common-eval-args.cc',
+  'editor-for.cc',
+  'installable-attr-path.cc',
+  'installable-derived-path.cc',
+  'installable-flake.cc',
+  'installable-value.cc',
+  'installables.cc',
+  'legacy.cc',
+  'markdown.cc',
+  'misc-store-flags.cc',
+  'network-proxy.cc',
+  'repl-interacter.cc',
+  'repl.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'built-path.hh',
+  'command-installable-value.hh',
+  'command.hh',
+  'common-eval-args.hh',
+  'compatibility-settings.hh',
+  'editor-for.hh',
+  'installable-attr-path.hh',
+  'installable-derived-path.hh',
+  'installable-flake.hh',
+  'installable-value.hh',
+  'installables.hh',
+  'legacy.hh',
+  'markdown.hh',
+  'misc-store-flags.hh',
+  'network-proxy.hh',
+  'repl-interacter.hh',
+  'repl.hh',
+)
+
+this_library = library(
+  'nixcmd',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libcmd/meson.options

@@ -0,0 +1,15 @@
+# vim: filetype=meson
+
+option(
+  'markdown',
+  type: 'feature',
+  description: 'Enable Markdown rendering in the Nix binary (requires lowdown)',
+)
+
+option(
+  'readline-flavor',
+  type : 'combo',
+  choices : ['editline', 'readline'],
+  value : 'editline',
+  description : 'Which library to use for nice line editing with the Nix language REPL',
+)

src/libcmd/network-proxy.cc

@@ -1,7 +1,6 @@
 #include "network-proxy.hh"
 
 #include <algorithm>
-#include <boost/algorithm/string.hpp>
 
 #include "environment-variables.hh"
 
@@ -13,7 +12,10 @@ static StringSet getAllVariables()
 {
     StringSet variables = lowercaseVariables;
     for (const auto & variable : lowercaseVariables) {
-        variables.insert(boost::to_upper_copy(variable));
+        std::string upperVariable;
+        std::transform(
+            variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); });
+        variables.insert(std::move(upperVariable));
     }
     return variables;
 }

src/libcmd/package.nix

@@ -0,0 +1,110 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+, nix-store
+, nix-fetchers
+, nix-expr
+, nix-flake
+, nix-main
+, editline
+, readline
+, lowdown
+, nlohmann_json
+
+# Configuration Options
+
+, version
+
+# Whether to enable Markdown rendering in the Nix binary.
+, enableMarkdown ? !stdenv.hostPlatform.isWindows
+
+# Which interactive line editor library to use for Nix's repl.
+#
+# Currently supported choices are:
+#
+# - editline (default)
+# - readline
+, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-cmd";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    ({ inherit editline readline; }.${readlineFlavor})
+  ] ++ lib.optional enableMarkdown lowdown;
+
+  propagatedBuildInputs = [
+    nix-util
+    nix-store
+    nix-fetchers
+    nix-expr
+    nix-flake
+    nix-main
+    nlohmann_json
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+    (lib.mesonEnable "markdown" enableMarkdown)
+    (lib.mesonOption "readline-flavor" readlineFlavor)
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  enableParallelBuilding = true;
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated
+  # to work with `strictDeps`.
+  strictDeps = true;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libcmd/repl-interacter.cc

@@ -18,7 +18,7 @@ extern "C" {
 #include "finally.hh"
 #include "repl-interacter.hh"
 #include "file-system.hh"
-#include "libcmd/repl.hh"
+#include "repl.hh"
 
 namespace nix {
 
@@ -73,7 +73,7 @@ static int listPossibleCallback(char * s, char *** avp)
 {
     auto possible = curRepl->completePrefix(s);
 
-    if (possible.size() > (INT_MAX / sizeof(char *)))
+    if (possible.size() > (std::numeric_limits<int>::max() / sizeof(char *)))
         throw Error("too many completions");
 
     int ac = 0;

src/libcmd/repl.cc

@@ -1,16 +1,14 @@
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
-#include <climits>
 
-#include "libcmd/repl-interacter.hh"
+#include "repl-interacter.hh"
 #include "repl.hh"
 
 #include "ansicolor.hh"
 #include "shared.hh"
+#include "config-global.hh"
 #include "eval.hh"
-#include "eval-cache.hh"
-#include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "attr-path.hh"
 #include "signals.hh"
@@ -28,12 +26,16 @@
 #include "markdown.hh"
 #include "local-fs-store.hh"
 #include "print.hh"
+#include "ref.hh"
+#include "value.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
 #include <gc/gc_cpp.h>
 #endif
 
+#include "strings.hh"
+
 namespace nix {
 
 /**
@@ -615,6 +617,38 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
     else if (command == ":doc") {
         Value v;
+
+        auto expr = parseString(arg);
+        std::string fallbackName;
+        PosIdx fallbackPos;
+        DocComment fallbackDoc;
+        if (auto select = dynamic_cast<ExprSelect *>(expr)) {
+            Value vAttrs;
+            auto name = select->evalExceptFinalSelect(*state, *env, vAttrs);
+            fallbackName = state->symbols[name];
+
+            state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation");
+            auto attrs = vAttrs.attrs();
+            assert(attrs);
+            auto attr = attrs->get(name);
+            if (!attr) {
+                // When missing, trigger the normal exception
+                // e.g. :doc builtins.foo
+                // behaves like
+                // nix-repl> builtins.foo      
+                // error: attribute 'foo' missing
+                evalString(arg, v);
+                assert(false);
+            }
+            if (attr->pos) {
+                fallbackPos = attr->pos;
+                fallbackDoc = state->getDocCommentForPos(fallbackPos);
+            }
+
+        } else {
+            evalString(arg, v);
+        }
+
         evalString(arg, v);
         if (auto doc = state->getDoc(v)) {
             std::string markdown;
@@ -632,6 +666,19 @@ ProcessLineResult NixRepl::processLine(std::string line)
             markdown += stripIndentation(doc->doc);
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
+        } else if (fallbackPos) {
+            std::stringstream ss;
+            ss << "Attribute `" << fallbackName << "`\n\n";
+            ss << "  … defined at " << state->positions[fallbackPos] << "\n\n";
+            if (fallbackDoc) {
+                ss << fallbackDoc.getInnerText(state->positions);
+            } else {
+                ss << "No documentation found.\n\n";
+            }
+
+            auto markdown = ss.str();
+            logger->cout(trim(renderMarkdownToTerminal(markdown)));
+
         } else
             throw Error("value does not have documentation");
     }
@@ -689,14 +736,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
     if (flakeRefS.empty())
         throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)");
 
-    auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true);
+    auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true);
     if (evalSettings.pureEval && !flakeRef.input.isLocked())
         throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
 
     Value v;
 
     flake::callFlake(*state,
-        flake::lockFlake(*state, flakeRef,
+        flake::lockFlake(flakeSettings, *state, flakeRef,
             flake::LockFlags {
                 .updateLockFile = false,
                 .useRegistries = !evalSettings.pureEval,

src/libexpr-c/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr-c/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr-c/local.mk

@@ -15,7 +15,7 @@ libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
                      $(INCLUDE_libstore) $(INCLUDE_libstorec) \
                      $(INCLUDE_libexpr) $(INCLUDE_libexprc)
 
-libexprc_LIBS = libutil libutilc libstore libstorec libexpr
+libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr
 
 libexprc_LDFLAGS += $(THREAD_LDFLAGS)
 

src/libexpr-c/meson.build

@@ -0,0 +1,91 @@
+project('nix-expr-c', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-expr'),
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util-c'),
+  dependency('nix-store-c'),
+]
+subdir('build-utils-meson/subprojects')
+
+# TODO rename, because it will conflict with downstream projects
+configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-expr.h',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+
+  # From C++ libraries, only for internals
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  '-include', 'config-expr.hh',
+
+  # From C libraries, for our public, installed headers too
+  '-include', 'config-util.h',
+  '-include', 'config-store.h',
+  '-include', 'config-expr.h',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'nix_api_expr.cc',
+  'nix_api_external.cc',
+  'nix_api_value.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'nix_api_expr.h',
+  'nix_api_external.h',
+  'nix_api_value.h',
+)
+
+# TODO move this header to libexpr, maybe don't use it in tests?
+headers += files('nix_api_expr_internal.h')
+
+subdir('build-utils-meson/export-all-symbols')
+
+this_library = library(
+  'nixexprc',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libexpr-c/nix_api_expr.cc

@@ -1,12 +1,11 @@
 #include <cstring>
-#include <iostream>
 #include <stdexcept>
 #include <string>
 
-#include "config.hh"
 #include "eval.hh"
+#include "eval-gc.hh"
 #include "globals.hh"
-#include "util.hh"
+#include "eval-settings.hh"
 
 #include "nix_api_expr.h"
 #include "nix_api_expr_internal.h"
@@ -15,10 +14,10 @@
 #include "nix_api_util.h"
 #include "nix_api_util_internal.h"
 
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
-#include <mutex>
+#  include <mutex>
-#define GC_INCLUDE_NEW 1
+#  define GC_INCLUDE_NEW 1
-#include "gc_cpp.h"
+#  include "gc_cpp.h"
 #endif
 
 nix_err nix_libexpr_init(nix_c_context * context)
@@ -106,7 +105,23 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
             for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
                 lookupPath.push_back(lookupPath_c[i]);
 
-        return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)};
+        void * p = ::operator new(
+            sizeof(EvalState),
+            static_cast<std::align_val_t>(alignof(EvalState)));
+        auto * p2 = static_cast<EvalState *>(p);
+        new (p) EvalState {
+            .fetchSettings = nix::fetchers::Settings{},
+            .settings = nix::EvalSettings{
+                nix::settings.readOnlyMode,
+            },
+            .state = nix::EvalState(
+                nix::LookupPath::parse(lookupPath),
+                store->ptr,
+                p2->fetchSettings,
+                p2->settings),
+        };
+        loadConfFile(p2->settings);
+        return p2;
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -116,7 +131,7 @@ void nix_state_free(EvalState * state)
     delete state;
 }
 
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
 std::unordered_map<
     const void *,
     unsigned int,
@@ -192,7 +207,7 @@ nix_err nix_value_decref(nix_c_context * context, nix_value *x)
 
 void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
 {
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
     GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
 #endif
 }

src/libexpr-c/nix_api_expr_internal.h

@@ -1,12 +1,16 @@
 #ifndef NIX_API_EXPR_INTERNAL_H
 #define NIX_API_EXPR_INTERNAL_H
 
+#include "fetch-settings.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "attr-set.hh"
 #include "nix_api_value.h"
 
 struct EvalState
 {
+    nix::fetchers::Settings fetchSettings;
+    nix::EvalSettings settings;
     nix::EvalState state;
 };
 

src/libexpr-c/nix_api_external.cc

@@ -14,7 +14,7 @@
 
 #include <nlohmann/json.hpp>
 
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
 # include "gc/gc.h"
 # define GC_INCLUDE_NEW 1
 # include "gc_cpp.h"
@@ -115,7 +115,7 @@ public:
     /**
      * Compare to another value of the same type.
      */
-    virtual bool operator==(const ExternalValueBase & b) const override
+    virtual bool operator==(const ExternalValueBase & b) const noexcept override
     {
         if (!desc.equal) {
             return false;
@@ -174,7 +174,7 @@ ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalV
         context->last_err_code = NIX_OK;
     try {
         auto ret = new
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
             (GC)
 #endif
                 NixCExternalValue(*desc, v);

src/libexpr-c/nix_api_value.cc

@@ -14,7 +14,7 @@
 #include "nix_api_value.h"
 #include "value/context.hh"
 
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
 #  include "gc/gc.h"
 #  define GC_INCLUDE_NEW 1
 #  include "gc_cpp.h"
@@ -131,7 +131,7 @@ PrimOp * nix_alloc_primop(
     try {
         using namespace std::placeholders;
         auto p = new
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
             (GC)
 #endif
                 nix::PrimOp{
@@ -383,7 +383,7 @@ nix_value * nix_get_attr_byidx(
     try {
         auto & v = check_value_in(value);
         const nix::Attr & a = (*v.attrs())[i];
-        *name = ((const std::string &) (state->state.symbols[a.name])).c_str();
+        *name = state->state.symbols[a.name].c_str();
         nix_gc_incref(nullptr, a.value);
         state->state.forceValue(*a.value, nix::noPos);
         return as_nix_value_ptr(a.value);
@@ -399,7 +399,7 @@ nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalSt
     try {
         auto & v = check_value_in(value);
         const nix::Attr & a = (*v.attrs())[i];
-        return ((const std::string &) (state->state.symbols[a.name])).c_str();
+        return state->state.symbols[a.name].c_str();
     }
     NIXC_CATCH_ERRS_NULL
 }

src/libexpr-c/package.nix

@@ -0,0 +1,78 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+
+, meson
+, ninja
+, pkg-config
+
+, nix-store-c
+, nix-expr
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-expr-c";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    # ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    (fileset.fileFilter (file: file.hasExt "h") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  propagatedBuildInputs = [
+    nix-store-c
+    nix-expr
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  enableParallelBuilding = true;
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  strictDeps = true;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libexpr/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr/attr-path.cc

@@ -76,7 +76,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
             if (!a) {
                 std::set<std::string> attrNames;
                 for (auto & attr : *v->attrs())
-                    attrNames.insert(state.symbols[attr.name]);
+                    attrNames.insert(std::string(state.symbols[attr.name]));
 
                 auto suggestions = Suggestions::bestMatches(attrNames, attr);
                 throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);

src/libexpr/attr-set.hh

@@ -5,7 +5,6 @@
 #include "symbol-table.hh"
 
 #include <algorithm>
-#include <optional>
 
 namespace nix {
 
@@ -28,9 +27,9 @@ struct Attr
     Attr(Symbol name, Value * value, PosIdx pos = noPos)
         : name(name), pos(pos), value(value) { };
     Attr() { };
-    bool operator < (const Attr & a) const
+    auto operator <=> (const Attr & a) const
     {
-        return name < a.name;
+        return name <=> a.name;
     }
 };
 

src/libexpr/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr/eval-cache.cc

@@ -95,7 +95,7 @@ struct AttrDb
     {
         try {
             auto state(_state->lock());
-            if (!failed)
+            if (!failed && state->txn->active)
                 state->txn->commit();
             state->txn.reset();
         } catch (...) {
@@ -225,7 +225,7 @@ struct AttrDb
                 (key.first)
                 (symbols[key.second])
                 (AttrType::ListOfStrings)
-                (concatStringsSep("\t", l)).exec();
+                (dropEmptyInitThenConcatStringsSep("\t", l)).exec();
 
             return state->db.getLastInsertedRowId();
         });
@@ -435,12 +435,12 @@ std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
 
 std::string AttrCursor::getAttrPathStr() const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
 }
 
 std::string AttrCursor::getAttrPathStr(Symbol name) const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
 }
 
 Value & AttrCursor::forceValue()
@@ -484,7 +484,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
     auto attrNames = getAttrs();
     std::set<std::string> strAttrNames;
     for (auto & name : attrNames)
-        strAttrNames.insert(root->state.symbols[name]);
+        strAttrNames.insert(std::string(root->state.symbols[name]));
 
     return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
 }

src/libexpr/eval-error.hh

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <algorithm>
-
 #include "error.hh"
 #include "pos-idx.hh"