Merge remote-tracking branch 'nixos/master'

.github/ISSUE_TEMPLATE/missing_documentation.md

@@ -23,7 +23,7 @@ assignees: ''
 - [ ] checked [open documentation issues and pull requests] for possible duplicates
 
 [latest Nix manual]: https://nixos.org/manual/nix/unstable/
-[source]: https://github.com/NixOS/nix/tree/master/doc/manual/src
+[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source
 [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation
 
 ## Priorities

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,22 @@
+<!--
+
+IMPORTANT
+
+Nix is a non-trivial project, so for your contribution to be successful,
+it really is important to follow the contributing guidelines:
+
+https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md
+
+Even if you've contributed to open source before, take a moment to read it,
+so you understand the process and the expectations.
+
+- what information to include in commit messages
+- proper attribution
+- volunteering contributions effectively
+- how to get help and our review process.
+
+-->
+
 # Motivation
 <!-- Briefly explain what the change is about and why it is desirable. -->
 

.github/labeler.yml

@@ -1,7 +1,7 @@
 "c api":
   - changed-files:
     - any-glob-to-any-file: "src/lib*-c/**/*"
-    - any-glob-to-any-file: "test/unit/**/nix_api_*"
+    - any-glob-to-any-file: "src/*test*/**/nix_api_*"
     - any-glob-to-any-file: "doc/external-api/**/*"
 
 "contributor-experience":
@@ -9,7 +9,7 @@
     - any-glob-to-any-file: "CONTRIBUTING.md"
     - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*"
     - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md"
-    - any-glob-to-any-file: "doc/manual/src/contributing/**"
+    - any-glob-to-any-file: "doc/manual/source/contributing/**"
 
 "documentation":
   - changed-files:

.github/workflows/backport.yml

@@ -1,32 +0,0 @@
-name: Backport
-on:
-  pull_request_target:
-    types: [closed, labeled]
-permissions:
-  contents: read
-jobs:
-  backport:
-    name: Backport Pull Request
-    permissions:
-      # for zeebe-io/backport-action
-      contents: write
-      pull-requests: write
-    if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name))
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          # required to find all branches
-          fetch-depth: 0
-      - name: Create backport PRs
-        # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v3.0.2
-        with:
-          # Config README: https://github.com/zeebe-io/backport-action#backport-action
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_workspace: ${{ github.workspace }}
-          pull_description: |-
-            Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}.
-          # should be kept in sync with `uses`
-          version: v0.0.5

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@V27
+    - uses: cachix/install-nix-action@v30
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -89,7 +89,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@V27
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - uses: cachix/cachix-action@v15
@@ -112,7 +112,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@V27
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -142,11 +142,11 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@V27
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
+    - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v15
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
@@ -214,4 +214,4 @@ jobs:
           path: flake-regressions/tests
       - uses: DeterminateSystems/nix-installer-action@main
       - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH scripts/flake-regressions.sh
+      - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh

.gitignore

@@ -23,17 +23,17 @@ perl/Makefile.config
 /doc/manual/conf-file.json
 /doc/manual/language.json
 /doc/manual/xp-features.json
-/doc/manual/src/SUMMARY.md
+/doc/manual/source/SUMMARY.md
-/doc/manual/src/SUMMARY-rl-next.md
+/doc/manual/source/SUMMARY-rl-next.md
-/doc/manual/src/store/types/*
+/doc/manual/source/store/types/*
-!/doc/manual/src/store/types/index.md.in
+!/doc/manual/source/store/types/index.md.in
-/doc/manual/src/command-ref/new-cli
+/doc/manual/source/command-ref/new-cli
-/doc/manual/src/command-ref/conf-file.md
+/doc/manual/source/command-ref/conf-file.md
-/doc/manual/src/command-ref/experimental-features-shortlist.md
+/doc/manual/source/command-ref/experimental-features-shortlist.md
-/doc/manual/src/contributing/experimental-feature-descriptions.md
+/doc/manual/source/contributing/experimental-feature-descriptions.md
-/doc/manual/src/language/builtins.md
+/doc/manual/source/language/builtins.md
-/doc/manual/src/language/builtin-constants.md
+/doc/manual/source/language/builtin-constants.md
-/doc/manual/src/release-notes/rl-next.md
+/doc/manual/source/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh
@@ -49,22 +49,22 @@ perl/Makefile.config
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
 /src/libexpr/tests
-/tests/unit/libexpr/libnixexpr-tests
+/src/libexpr-tests/libnixexpr-tests
 
 # /src/libfetchers
-/tests/unit/libfetchers/libnixfetchers-tests
+/src/libfetchers-tests/libnixfetchers-tests
 
 # /src/libflake
-/tests/unit/libflake/libnixflake-tests
+/src/libflake-tests/libnixflake-tests
 
 # /src/libstore/
 *.gen.*
 /src/libstore/tests
-/tests/unit/libstore/libnixstore-tests
+/src/libstore-tests/libnixstore-tests
 
 # /src/libutil/
 /src/libutil/tests
-/tests/unit/libutil/libnixutil-tests
+/src/libutil-tests/libnixutil-tests
 
 /src/nix/nix
 

.mergify.yml

@@ -0,0 +1,92 @@
+queue_rules:
+  - name: default
+    # all required tests need to go here
+    merge_conditions:
+      - check-success=installer
+      - check-success=installer_test (macos-latest)
+      - check-success=installer_test (ubuntu-latest)
+      - check-success=tests (macos-latest)
+      - check-success=tests (ubuntu-latest)
+      - check-success=vm_tests
+    merge_method: rebase
+    batch_size: 5
+
+pull_request_rules:
+  - name: merge using the merge queue
+    conditions:
+      - base~=master|.+-maintenance
+      - label~=merge-queue|dependencies
+    actions:
+      queue: {}
+
+# The rules below will first create backport pull requests and put those in a merge queue.
+
+  - name: backport patches to 2.18
+    conditions:
+      - label=backport 2.18-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.18-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.19
+    conditions:
+      - label=backport 2.19-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.19-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.20
+    conditions:
+      - label=backport 2.20-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.20-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.21
+    conditions:
+      - label=backport 2.21-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.21-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.22
+    conditions:
+      - label=backport 2.22-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.22-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.23
+    conditions:
+      - label=backport 2.23-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.23-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.24
+    conditions:
+      - label=backport 2.24-maintenance
+    actions:
+      backport:
+        branches:
+          - "2.24-maintenance"
+        labels:
+          - merge-queue

CONTRIBUTING.md

@@ -79,7 +79,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
      - Functional tests – [`tests/functional/**.sh`](./tests/functional)
      - Unit tests – [`src/*/tests`](./src/)
      - Integration tests – [`tests/nixos/*`](./tests/nixos)
-   - [ ] User documentation in the [manual](./doc/manual/src)
+   - [ ] User documentation in the [manual](./doc/manual/source)
    - [ ] API documentation in header files
    - [ ] Code and comments are self-explanatory
    - [ ] Commit message explains **why** the change was made
@@ -90,11 +90,11 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
 ## Making changes to the Nix manual
 
 The Nix reference manual is hosted on https://nixos.org/manual/nix.
-The underlying source files are located in [`doc/manual/src`](./doc/manual/src).
+The underlying source files are located in [`doc/manual/source`](./doc/manual/source).
 For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files)
 For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html).
 
 ## Getting help
 
 Whenever you're stuck or do not know how to proceed, you can always ask for help.
-The appropriate channels to do so can be found on the [NixOS Community](https://nixos.org/community/) page.
+We invite you to use our [Matrix room](https://matrix.to/#/#nix-dev:nixos.org) to ask questions.

HACKING.md

@@ -0,0 +1 @@
+doc/manual/source/development/building.md

Makefile

@@ -38,6 +38,18 @@ makefiles += \
 endif
 endif
 
+ifeq ($(ENABLE_UNIT_TESTS), yes)
+makefiles += \
+  src/libutil-tests/local.mk \
+  src/libutil-test-support/local.mk \
+  src/libstore-tests/local.mk \
+  src/libstore-test-support/local.mk \
+  src/libfetchers-tests/local.mk \
+  src/libexpr-tests/local.mk \
+  src/libexpr-test-support/local.mk \
+  src/libflake-tests/local.mk
+endif
+
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 ifdef HOST_UNIX
 makefiles += \
@@ -79,6 +91,7 @@ ifdef HOST_WINDOWS
   #
   # TODO do not do this, and instead do fine-grained export annotations.
   GLOBAL_LDFLAGS += -Wl,--export-all-symbols
+  GLOBAL_CXXFLAGS += -D_WIN32_WINNT=0x0602
 endif
 
 GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -Werror=suggest-override -include $(buildprefix)config.h -std=c++2a -I src
@@ -92,6 +105,13 @@ include mk/lib.mk
 # These must be defined after `mk/lib.mk`. Otherwise the first rule
 # incorrectly becomes the default target.
 
+ifneq ($(ENABLE_UNIT_TESTS), yes)
+.PHONY: check
+check:
+	@echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'."
+	@exit 1
+endif
+
 ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 .PHONY: installcheck
 installcheck:

Makefile.config.in

@@ -12,6 +12,7 @@ ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_DOC_GEN = @ENABLE_DOC_GEN@
 ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@
 ENABLE_S3 = @ENABLE_S3@
+ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@

build-utils-meson/export/meson.build

@@ -10,6 +10,7 @@ foreach dep : deps_public_subproject
 endforeach
 requires_public += deps_public
 
+extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {})
 import('pkgconfig').generate(
   this_library,
   filebase : meson.project_name(),
@@ -20,6 +21,7 @@ import('pkgconfig').generate(
   requires : requires_public,
   requires_private : requires_private,
   libraries_private : libraries_private,
+  variables : extra_pkg_config_variables,
 )
 
 meson.override_dependency(meson.project_name(), declare_dependency(
@@ -27,4 +29,5 @@ meson.override_dependency(meson.project_name(), declare_dependency(
   link_with : this_library,
   compile_args : ['-std=c++2a'],
   dependencies : deps_public_subproject + deps_public,
+  variables : extra_pkg_config_variables,
 ))

build-utils-meson/libatomic/meson.build

@@ -0,0 +1,8 @@
+
+# Check if -latomic is needed
+# This is needed for std::atomic on some platforms
+# We did not manage to test this reliably on all platforms, so we hardcode
+# it for now.
+if host_machine.cpu_family() == 'arm'
+  deps_other += cxx.find_library('atomic')
+endif

build-utils-meson/windows-version/meson.build

@@ -0,0 +1,6 @@
+if host_machine.system() == 'windows'
+  # https://learn.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt?view=msvc-170
+  # #define _WIN32_WINNT_WIN8 0x0602
+  # We currently don't use any API which requires higher than this.
+  add_project_arguments([ '-D_WIN32_WINNT=0x0602' ], language: 'cpp')
+endif

configure.ac

@@ -62,12 +62,16 @@ AC_CHECK_TOOL([AR], [ar])
 AC_SYS_LARGEFILE
 
 
-# Solaris-specific stuff.
+# OS-specific stuff.
 case "$host_os" in
   solaris*)
     # Solaris requires -lsocket -lnsl for network functions
     LDFLAGS="-lsocket -lnsl $LDFLAGS"
     ;;
+  darwin*)
+    # Need to link to libsandbox.
+    LDFLAGS="-lsandbox $LDFLAGS"
+    ;;
 esac
 
 
@@ -86,12 +90,13 @@ static char buf[1024];]],
 AC_LANG_POP(C++)
 
 
-AC_CHECK_FUNCS([statvfs pipe2])
+AC_CHECK_FUNCS([statvfs pipe2 close_range])
 
 
-# Check for lutimes, optionally used for changing the mtime of
+# Check for lutimes and utimensat, optionally used for changing the
-# symlinks.
+# mtime of symlinks.
-AC_CHECK_FUNCS([lutimes])
+AC_CHECK_DECLS([AT_SYMLINK_NOFOLLOW], [], [], [[#include <fcntl.h>]])
+AC_CHECK_FUNCS([lutimes utimensat])
 
 
 # Check whether the store optimiser can optimise symlinks.
@@ -141,6 +146,18 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
   ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
 AC_SUBST(ENABLE_BUILD)
 
+# Building without unit tests is useful for bootstrapping with a smaller footprint
+# or running the tests in a separate derivation. Otherwise, we do compile and
+# run them.
+
+AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]),
+  ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD)
+AC_SUBST(ENABLE_UNIT_TESTS)
+
+AS_IF(
+  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"],
+  [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])])
+
 AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]),
   ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes)
 AC_SUBST(ENABLE_FUNCTIONAL_TESTS)
@@ -158,6 +175,10 @@ AS_IF(
   [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"],
   [NEED_PROG(jq, jq)])
 
+AS_IF(
+  [test "$ENABLE_DOC_GEN" == "yes"],
+  [NEED_PROG(man, man)])
+
 AS_IF([test "$ENABLE_BUILD" == "yes"],[
 
 # Look for boost, a required dependency.
@@ -346,6 +367,16 @@ if test "$gc" = yes; then
   CFLAGS="$old_CFLAGS"
 fi
 
+AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[
+
+# Look for gtest.
+PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main])
+
+# Look for rapidcheck.
+PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest])
+
+])
+
 # Look for nlohmann/json.
 PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 

doc/manual/.version

@@ -0,0 +1 @@
+../../.version

doc/manual/book.toml

@@ -1,5 +1,6 @@
 [book]
 title = "Nix Reference Manual"
+src = "source"
 
 [output.html]
 additional-css = ["custom.css"]
@@ -7,9 +8,21 @@ additional-js = ["redirects.js"]
 edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
 git-repository-url = "https://github.com/NixOS/nix"
 
+# Handles replacing @docroot@ with a path to ./source relative to that markdown file,
+# {{#include handlebars}}, and the @generated@ syntax used within these. it mostly
+# but not entirely replaces the links preprocessor (which we cannot simply use due
+# to @generated@ files living in a different directory to make meson happy). we do
+# not want to disable the links preprocessor entirely though because that requires
+# disabling *all* built-in preprocessors and selectively reenabling those we want.
+[preprocessor.substitute]
+command = "python3 ./substitute.py"
+before = ["anchors", "links"]
+
 [preprocessor.anchors]
 renderers = ["html"]
-command = "jq --from-file doc/manual/anchors.jq"
+command = "jq --from-file ./anchors.jq"
+
+[output.markdown]
 
 [output.linkcheck]
 # no Internet during the build (in the sandbox)

doc/manual/generate-deps.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import glob
+import sys
+
+# meson expects makefile-style dependency declarations, i.e.
+#
+#   target: dependency...
+#
+# meson seems to pass depfiles straight on to ninja even though
+# it also parses the file itself (or at least has code to do so
+# in its tree), so we must live by ninja's rules: only slashes,
+# spaces and octothorpes can be escaped, anything else is taken
+# literally. since the rules for these aren't even the same for
+# all three we will just fail when we encounter any of them (if
+# asserts are off for some reason the depfile will likely point
+# to nonexistant paths, making everything phony and thus fine.)
+for path in glob.glob(sys.argv[1] + '/**', recursive=True):
+    assert '\\' not in path
+    assert ' ' not in path
+    assert '#' not in path
+    print("ignored:", path)

doc/manual/generate-store-types.nix

@@ -21,7 +21,7 @@ let
 
   "index.md" = replaceStrings
     [ "@store-types@" ] [ index ]
-    (readFile ./src/store/types/index.md.in);
+    (readFile ./source/store/types/index.md.in);
 
   tableOfContents =
     let

doc/manual/local.mk

@@ -4,8 +4,8 @@
 doc_nix = $(nix_PATH)
 
 MANUAL_SRCS := \
-	$(call rwildcard, $(d)/src, *.md) \
+	$(call rwildcard, $(d)/source, *.md) \
-	$(call rwildcard, $(d)/src, */*.md)
+	$(call rwildcard, $(d)/source, */*.md)
 
 man-pages := $(foreach n, \
 	nix-env.1 nix-store.1 \
@@ -18,11 +18,11 @@ man-pages := $(foreach n, \
 , $(d)/$(n))
 
 # man pages for subcommands
-# convert from `$(d)/src/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1`
+# convert from `$(d)/source/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1`
 # FIXME: unify with how nix3-cli man pages are generated
 man-pages += $(foreach subcommand, \
-	$(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/src/command-ref/nix-*/*.md)), \
+	$(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/source/command-ref/nix-*/*.md)), \
-	$(d)/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand)))))
+	$(d)/$(subst /,-,$(subst $(d)/source/command-ref/,,$(subst .md,.1,$(subcommand)))))
 
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
@@ -49,11 +49,11 @@ define process-includes
 	done < <(grep '{{#include' $(1))
 endef
 
-$(d)/nix-env-%.1: $(d)/src/command-ref/nix-env/%.md
+$(d)/nix-env-%.1: $(d)/source/command-ref/nix-env/%.md
 	@printf "Title: %s\n\n" "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" > $^.tmp
 	$(render-subcommand)
 
-$(d)/nix-store-%.1: $(d)/src/command-ref/nix-store/%.md
+$(d)/nix-store-%.1: $(d)/source/command-ref/nix-store/%.md
 	@printf -- 'Title: %s\n\n' "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" > $^.tmp
 	$(render-subcommand)
 
@@ -69,50 +69,50 @@ define render-subcommand
 endef
 
 
-$(d)/%.1: $(d)/src/command-ref/%.md
+$(d)/%.1: $(d)/source/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
 	@cat $^ >> $^.tmp
 	@$(call process-includes,$^,$^.tmp)
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/%.8: $(d)/src/command-ref/%.md
+$(d)/%.8: $(d)/source/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp
 	@cat $^ >> $^.tmp
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=8 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
+$(d)/nix.conf.5: $(d)/source/command-ref/conf-file.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
 	@$(call process-includes,$^,$^.tmp)
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
+$(d)/nix-profiles.5: $(d)/source/command-ref/files/profiles.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md
+$(d)/source/SUMMARY.md: $(d)/source/SUMMARY.md.in $(d)/source/SUMMARY-rl-next.md $(d)/source/store/types $(d)/source/command-ref/new-cli $(d)/source/development/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix)
+$(d)/source/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/source/store/types/index.md.in $(doc_nix)
 	@# FIXME: build out of tree!
 	@rm -rf $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores'
 	@# do not destroy existing contents
 	@mv $@.tmp/* $@/
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
+$(d)/source/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix)
+$(d)/source/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/source/command-ref/conf-file-prefix.md $(d)/source/command-ref/experimental-features-shortlist.md $(doc_nix)
-	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
+	@cat doc/manual/source/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
@@ -124,12 +124,12 @@ $(d)/conf-file.json: $(doc_nix)
 	$(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
+$(d)/source/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
+$(d)/source/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
@@ -138,10 +138,10 @@ $(d)/xp-features.json: $(doc_nix)
 	$(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
+$(d)/source/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/source/language/builtins-prefix.md $(doc_nix)
-	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
+	@cat doc/manual/source/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
-	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
+	@cat doc/manual/source/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/language.json: $(doc_nix)
@@ -149,7 +149,7 @@ $(d)/language.json: $(doc_nix)
 	@mv $@.tmp $@
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)
-$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
+$(d)/source/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
 	@if type -p changelog-d > /dev/null; then \
 		echo "  GEN   " $@; \
 		changelog-d doc/manual/rl-next > $@; \
@@ -158,7 +158,7 @@ $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
 		true > $@; \
 	fi
 
-$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+$(d)/source/SUMMARY-rl-next.md: $(d)/source/release-notes/rl-next.md
 	$(trace-gen) true
 	@if [ -s $< ]; then \
 		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
@@ -194,9 +194,9 @@ $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages
 	@mkdir -p $(DESTDIR)$$(dirname $@)
 	$(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@)
 
-doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
+doc/manual/generated/man1/nix3-manpages: $(d)/source/command-ref/new-cli
 	@mkdir -p $(DESTDIR)$$(dirname $@)
-	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
+	$(trace-gen) for i in doc/manual/source/command-ref/new-cli/*.md; do \
 		name=$$(basename $$i .md); \
 		tmpFile=$$(mktemp); \
 		if [[ $$name = SUMMARY ]]; then continue; fi; \
@@ -211,7 +211,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/source/SUMMARY.md $(d)/source/store/types $(d)/source/command-ref/new-cli $(d)/source/development/experimental-feature-descriptions.md $(d)/source/command-ref/conf-file.md $(d)/source/language/builtins.md $(d)/source/release-notes/rl-next.md $(d)/source/figures $(d)/source/favicon.png $(d)/source/favicon.svg
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \
@@ -219,12 +219,17 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/
 			$(call process-includes,$$file,$$file); \
 		done; \
 		find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \
-			docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \
+			docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/source)"; \
 			sed -i "s,@docroot@,$$docroot,g" "$$file"; \
 		done; \
 		set -euo pipefail; \
-		RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
+		( \
-			| { grep -Fv "because fragment resolution isn't implemented" || :; }; \
+		    cd "$$tmp/manual"; \
+		    RUST_LOG=warn \
+		        MDBOOK_SUBSTITUTE_SEARCH=$(d)/source \
+		        mdbook build -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
+			    | { grep -Fv "because fragment resolution isn't implemented" || :; } \
+		); \
 		rm -rf "$$tmp/manual"
 	@rm -rf $(DESTDIR)$(docdir)/manual
 	@mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual

doc/manual/meson.build

@@ -0,0 +1,353 @@
+project('nix-manual',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+nix = find_program('nix', native : true)
+
+mdbook = find_program('mdbook', native : true)
+bash = find_program('bash', native : true)
+
+pymod = import('python')
+python = pymod.find_installation('python3')
+
+nix_env_for_docs = {
+  'HOME': '/dummy',
+  'NIX_CONF_DIR': '/dummy',
+  'NIX_SSL_CERT_FILE': '/dummy/no-ca-bundle.crt',
+  'NIX_STATE_DIR': '/dummy',
+  'NIX_CONFIG': 'cores = 0',
+}
+
+nix_for_docs = [nix, '--experimental-features', 'nix-command']
+nix_eval_for_docs_common = nix_for_docs + [
+  'eval',
+  '-I', 'nix=' + meson.current_source_dir(),
+  '--store', 'dummy://',
+  '--impure',
+]
+nix_eval_for_docs = nix_eval_for_docs_common + '--raw'
+
+conf_file_json = custom_target(
+  command : nix_for_docs + ['config', 'show', '--json'],
+  capture : true,
+  output : 'conf-file.json',
+  env : nix_env_for_docs,
+)
+
+language_json = custom_target(
+  command: [nix, '__dump-language'],
+  output : 'language.json',
+  capture : true,
+  env : nix_env_for_docs,
+)
+
+nix3_cli_json = custom_target(
+  command : [nix, '__dump-cli'],
+  capture : true,
+  output : 'nix.json',
+  env : nix_env_for_docs,
+)
+
+generate_manual_deps = files(
+  'generate-deps.py',
+)
+
+# Generates types
+subdir('source/store')
+# Generates builtins.md and builtin-constants.md.
+subdir('source/language')
+# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md.
+subdir('source/command-ref')
+# Generates experimental-feature-descriptions.md.
+subdir('source/development')
+# Generates rl-next-generated.md.
+subdir('source/release-notes')
+subdir('source')
+
+# Hacky way to figure out if `nix` is an `ExternalProgram` or
+# `Exectuable`. Only the latter can occur in custom target input lists.
+if nix.full_path().startswith(meson.build_root())
+  nix_input = nix
+else
+  nix_input = []
+endif
+
+manual = custom_target(
+  'manual',
+  command : [
+    bash,
+    '-euo', 'pipefail',
+    '-c',
+    '''
+        @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
+        @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
+        rsync -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
+        (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
+        rm -rf @2@/manual
+        mv @2@/html @2@/manual
+        find @2@/manual -iname meson.build -delete
+    '''.format(
+      python.full_path(),
+      mdbook.full_path(),
+      meson.current_build_dir(),
+    ),
+  ],
+  input : [
+    generate_manual_deps,
+    'substitute.py',
+    'book.toml',
+    'anchors.jq',
+    'custom.css',
+    nix3_cli_files,
+    experimental_features_shortlist_md,
+    experimental_feature_descriptions_md,
+    types_dir,
+    conf_file_md,
+    builtins_md,
+    rl_next_generated,
+    summary_rl_next,
+    nix_input,
+  ],
+  output : [
+    'manual',
+    'markdown',
+  ],
+  depfile : 'manual.d',
+  env : {
+    'RUST_LOG': 'info',
+    'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'source',
+  },
+)
+manual_html = manual[0]
+manual_md = manual[1]
+
+install_subdir(
+  manual_html.full_path(),
+  install_dir : get_option('datadir') / 'doc/nix',
+)
+
+nix_nested_manpages = [
+  [ 'nix-env',
+    [
+      'delete-generations',
+      'install',
+      'list-generations',
+      'query',
+      'rollback',
+      'set-flag',
+      'set',
+      'switch-generation',
+      'switch-profile',
+      'uninstall',
+      'upgrade',
+    ],
+  ],
+  [ 'nix-store',
+    [
+      'add-fixed',
+      'add',
+      'delete',
+      'dump-db',
+      'dump',
+      'export',
+      'gc',
+      'generate-binary-cache-key',
+      'import',
+      'load-db',
+      'optimise',
+      'print-env',
+      'query',
+      'read-log',
+      'realise',
+      'repair-path',
+      'restore',
+      'serve',
+      'verify',
+      'verify-path',
+    ],
+  ],
+]
+
+foreach command : nix_nested_manpages
+  foreach page : command[1]
+    title = command[0] + ' --' + page
+    section = '1'
+    custom_target(
+      command : [
+        bash,
+        files('./render-manpage.sh'),
+        '--out-no-smarty',
+        title,
+        section,
+        '@INPUT0@/command-ref' / command[0] / (page + '.md'),
+        '@OUTPUT0@',
+      ],
+      input : [
+        manual_md,
+        nix_input,
+      ],
+      output : command[0] + '-' + page + '.1',
+      install : true,
+      install_dir : get_option('mandir') / 'man1',
+    )
+  endforeach
+endforeach
+
+nix3_manpages = [
+  'nix3-build',
+  'nix3-bundle',
+  'nix3-config',
+  'nix3-config-show',
+  'nix3-copy',
+  'nix3-daemon',
+  'nix3-derivation-add',
+  'nix3-derivation',
+  'nix3-derivation-show',
+  'nix3-develop',
+  #'nix3-doctor',
+  'nix3-edit',
+  'nix3-eval',
+  'nix3-flake-archive',
+  'nix3-flake-check',
+  'nix3-flake-clone',
+  'nix3-flake-info',
+  'nix3-flake-init',
+  'nix3-flake-lock',
+  'nix3-flake',
+  'nix3-flake-metadata',
+  'nix3-flake-new',
+  'nix3-flake-prefetch',
+  'nix3-flake-show',
+  'nix3-flake-update',
+  'nix3-fmt',
+  'nix3-hash-file',
+  'nix3-hash',
+  'nix3-hash-path',
+  'nix3-hash-to-base16',
+  'nix3-hash-to-base32',
+  'nix3-hash-to-base64',
+  'nix3-hash-to-sri',
+  'nix3-help',
+  'nix3-help-stores',
+  'nix3-key-convert-secret-to-public',
+  'nix3-key-generate-secret',
+  'nix3-key',
+  'nix3-log',
+  'nix3-nar-cat',
+  'nix3-nar-dump-path',
+  'nix3-nar-ls',
+  'nix3-nar',
+  'nix3-path-info',
+  'nix3-print-dev-env',
+  'nix3-profile-diff-closures',
+  'nix3-profile-history',
+  'nix3-profile-install',
+  'nix3-profile-list',
+  'nix3-profile',
+  'nix3-profile-remove',
+  'nix3-profile-rollback',
+  'nix3-profile-upgrade',
+  'nix3-profile-wipe-history',
+  'nix3-realisation-info',
+  'nix3-realisation',
+  'nix3-registry-add',
+  'nix3-registry-list',
+  'nix3-registry',
+  'nix3-registry-pin',
+  'nix3-registry-remove',
+  'nix3-repl',
+  'nix3-run',
+  'nix3-search',
+  #'nix3-shell',
+  'nix3-store-add-file',
+  'nix3-store-add-path',
+  'nix3-store-cat',
+  'nix3-store-copy-log',
+  'nix3-store-copy-sigs',
+  'nix3-store-delete',
+  'nix3-store-diff-closures',
+  'nix3-store-dump-path',
+  'nix3-store-gc',
+  'nix3-store-ls',
+  'nix3-store-make-content-addressed',
+  'nix3-store',
+  'nix3-store-optimise',
+  'nix3-store-path-from-hash-part',
+  'nix3-store-ping',
+  'nix3-store-prefetch-file',
+  'nix3-store-repair',
+  'nix3-store-sign',
+  'nix3-store-verify',
+  'nix3-upgrade-nix',
+  'nix3-why-depends',
+  'nix',
+]
+
+foreach page : nix3_manpages
+  section = '1'
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      page,
+      section,
+      '@INPUT1@/command-ref/new-cli/@0@.md'.format(page),
+      '@OUTPUT@',
+    ],
+    input : [
+      files('./render-manpage.sh'),
+      manual_md,
+      nix_input,
+    ],
+    output : page + '.1',
+    install : true,
+    install_dir : get_option('mandir') / 'man1',
+  )
+endforeach
+
+nix_manpages = [
+  [ 'nix-env', 1 ],
+  [ 'nix-store', 1 ],
+  [ 'nix-build', 1 ],
+  [ 'nix-shell', 1 ],
+  [ 'nix-instantiate', 1 ],
+  [ 'nix-collect-garbage', 1 ],
+  [ 'nix-prefetch-url', 1 ],
+  [ 'nix-channel', 1 ],
+  [ 'nix-hash', 1 ],
+  [ 'nix-copy-closure', 1 ],
+  [ 'nix.conf', 5, conf_file_md.full_path() ],
+  [ 'nix-daemon', 8 ],
+  [ 'nix-profiles', 5, 'files/profiles.md' ],
+]
+
+foreach entry : nix_manpages
+  title = entry[0]
+  # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md,
+  # rather than a stem identical to its mdbook source.
+  # Therefore we use an optional third element of this array to override the name pattern
+  md_file = entry.get(2, title + '.md')
+  section = entry[1].to_string()
+  md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file)
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      title,
+      section,
+      md_file_resolved,
+      '@OUTPUT@',
+    ],
+    input : [
+      files('./render-manpage.sh'),
+      manual_md,
+      entry.get(3, []),
+      nix_input,
+    ],
+    output : '@0@.@1@'.format(entry[0], entry[1]),
+    install : true,
+    install_dir : get_option('mandir') / 'man@0@'.format(entry[1]),
+  )
+endforeach

doc/manual/package.nix

@@ -0,0 +1,71 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, lowdown
+, mdbook
+, mdbook-linkcheck
+, jq
+, python3
+, rsync
+, nix-cli
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-manual";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.difference
+    (fileset.unions [
+      ../../.version
+      # Too many different types of files to filter for now
+      ../../doc/manual
+      ./.
+    ])
+    # Do a blacklist instead
+    ../../doc/manual/package.nix;
+
+  # TODO the man pages should probably be separate
+  outputs = [ "out" "man" ];
+
+  # Hack for sake of the dev shell
+  passthru.externalNativeBuildInputs = [
+    meson
+    ninja
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+    jq
+    python3
+    rsync
+  ];
+
+  nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [
+    nix-cli
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''$out/nix-support
+    echo "doc manual ''$out/share/doc/nix/manual" >> ''$out/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

doc/manual/redirects.js

@@ -1,7 +1,7 @@
 // redirect rules for URL fragments (client-side) to prevent link rot.
 // this must be done on the client side, as web servers do not see the fragment part of the URL.
 // it will only work with JavaScript enabled in the browser, but this is the best we can do here.
-// see src/_redirects for path redirects (server-side)
+// see source/_redirects for path redirects (server-side)
 
 // redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.

doc/manual/remove_before_wrapper.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import shutil
+import typing as t
+
+def main():
+    if len(sys.argv) < 4 or '--' not in sys.argv:
+        print("Usage: remove-before-wrapper <output> -- <nix command...>")
+        sys.exit(1)
+
+    # Extract the parts
+    output: str = sys.argv[1]
+    nix_command_idx: int = sys.argv.index('--') + 1
+    nix_command: t.List[str] = sys.argv[nix_command_idx:]
+
+    output_temp: str = output + '.tmp'
+
+    # Remove the output and temp output in case they exist
+    shutil.rmtree(output, ignore_errors=True)
+    shutil.rmtree(output_temp, ignore_errors=True)
+
+    # Execute nix command with `--write-to` tempary output
+    nix_command_write_to = nix_command + ['--write-to', output_temp]
+    subprocess.run(nix_command_write_to, check=True)
+
+    # Move the temporary output to the intended location
+    os.rename(output_temp, output)
+
+if __name__ == "__main__":
+    main()

doc/manual/render-manpage.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+lowdown_args=
+
+if [ "$1" = --out-no-smarty ]; then
+    lowdown_args=--out-no-smarty
+    shift
+fi
+
+[ "$#" = 4 ] || {
+    echo "wrong number of args passed" >&2
+    exit 1
+}
+
+title="$1"
+section="$2"
+infile="$3"
+outfile="$4"
+
+(
+    printf "Title: %s\n\n" "$title"
+    cat "$infile"
+) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile"

doc/manual/rl-next/add-nix-state-home.md

@@ -0,0 +1,14 @@
+---
+synopsis: Use envvars NIX_CACHE_HOME, NIX_CONFIG_HOME, NIX_DATA_HOME, NIX_STATE_HOME if defined
+prs: [11351]
+---
+
+Added new environment variables:
+
+- `NIX_CACHE_HOME`
+- `NIX_CONFIG_HOME`
+- `NIX_DATA_HOME`
+- `NIX_STATE_HOME`
+
+Each, if defined, takes precedence over the corresponding [XDG environment variable](@docroot@/command-ref/env-common.md#xdg-base-directories).
+This provides more fine-grained control over where Nix looks for files, and allows to have a stand-alone Nix environment, which only uses files in a specific directory, and doesn't interfere with the user environment.

doc/manual/rl-next/build-hook-default.md

@@ -16,7 +16,7 @@ This has a small adverse affect on remote building --- the `build-remote` execut
 This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work.
 
 Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221).
-There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder.
+There should simply be no need to have an extra, intermediate layer of remote-procedure-calling when we want to connect to a remote builder.
-The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/) i.e. `Store` subclass. <!-- we normally don't mention classes, but consider that this release note is about a library use case -->
+The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/index.md) i.e. `Store` subclass. <!-- we normally don't mention classes, but consider that this release note is about a library use case -->
 
-The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above.
+The Perl bindings no longer expose `getBinDir` either, since the underlying C++ libraries those bindings wrap no longer know the location of installed binaries as described above.

doc/manual/rl-next/filesystem-errors.md

@@ -0,0 +1,14 @@
+---
+synopsis: wrap filesystem exceptions more correctly
+issues: []
+prs: [11378]
+---
+
+
+With the switch to `std::filesystem` in different places, Nix started to throw `std::filesystem::filesystem_error` in many places instead of its own exceptions.
+
+This lead to no longer generating error traces, for example when listing a non-existing directory, and can also lead to crashes inside the Nix REPL.
+
+This version catches these types of exception correctly and wrap them into Nix's own exeception type.
+
+Author: [**@Mic92**](https://github.com/Mic92)

doc/manual/rl-next/fsync-store-paths.md

@@ -0,0 +1,9 @@
+---
+synopsis: Add setting `fsync-store-paths`
+issues: [1218]
+prs: [7126]
+---
+
+Nix now has a setting `fsync-store-paths` that ensures that new store paths are durably written to disk before they are registered as "valid" in Nix's database. This can prevent Nix store corruption if the system crashes or there is a power loss. This setting defaults to `false`.
+
+Author: [**@squalus**](https://github.com/squalus)

doc/manual/rl-next/nix-fmt-default-argument.md

@@ -0,0 +1,17 @@
+---
+synopsis: Removing the default argument passed to the `nix fmt` formatter
+issues: []
+prs: [11438]
+---
+
+The underlying formatter no longer receives the ". " default argument when `nix fmt` is called with no arguments.
+
+This change was necessary as the formatter wasn't able to distinguish between
+a user wanting to format the current folder with `nix fmt .` or the generic
+`nix fmt`.
+
+The default behaviour is now the responsibility of the formatter itself, and
+allows tools such as treefmt to format the whole tree instead of only the
+current directory and below.
+
+Author: [**@zimbatm**](https://github.com/zimbatm)

doc/manual/rl-next/no-flake-substitution.md

@@ -0,0 +1,8 @@
+---
+synopsis: Flakes are no longer substituted
+prs: [10612]
+---
+
+Nix will no longer attempt to substitute the source code of flakes from a binary cache. This functionality was broken because it could lead to different evaluation results depending on whether the flake was available in the binary cache, or even depending on whether the flake was already in the local store.
+
+Author: [**@edolstra**](https://github.com/edolstra)

doc/manual/rl-next/verify-tls.md

@@ -0,0 +1,8 @@
+---
+synopsis: "`<nix/fetchurl.nix>` uses TLS verification"
+prs: [11585]
+---
+
+Previously `<nix/fetchurl.nix>` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `<nix/fetchurl.nix>` will now fail if you're fetching from a HTTPS server that does not have a valid certificate.
+
+`<nix/fetchurl.nix>` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue.

doc/manual/source/SUMMARY.md.in

@@ -121,6 +121,7 @@
 - [Development](development/index.md)
   - [Building](development/building.md)
   - [Testing](development/testing.md)
+  - [Debugging](development/debugging.md)
   - [Documentation](development/documentation.md)
   - [CLI guideline](development/cli-guideline.md)
   - [JSON guideline](development/json-guideline.md)

doc/manual/source/advanced-topics/distributed-builds.md

@@ -0,0 +1,108 @@
+# Remote Builds
+
+A local Nix installation can forward Nix builds to other machines,
+this allows multiple builds to be performed in parallel.
+
+Remote builds also allow Nix to perform multi-platform builds in a
+semi-transparent way. For example, if you perform a build for a
+`x86_64-darwin` on an `i686-linux` machine, Nix can automatically
+forward the build to a `x86_64-darwin` machine, if one is available.
+
+## Requirements
+
+For a local machine to forward a build to a remote machine, the remote machine must:
+
+- Have Nix installed
+- Be running an SSH server, e.g. `sshd`
+- Be accessible via SSH from the local machine over the network
+- Have the local machine's public SSH key in `/etc/ssh/authorized_keys.d/<username>`
+- Have the username of the SSH user in the `trusted-users` setting in `nix.conf`
+
+## Testing
+
+To test connecting to a remote Nix instance (in this case `mac`), run:
+
+```console
+nix store info --store ssh://username@mac
+```
+
+To specify an SSH identity file as part of the remote store URI add a
+query paramater, e.g.
+
+```console
+nix store info --store ssh://username@mac?ssh-key=/home/alice/my-key
+```
+
+Since builds should be non-interactive, the key should not have a
+passphrase. Alternatively, you can load identities ahead of time into
+`ssh-agent` or `gpg-agent`.
+
+In a multi-user installation (default), builds are executed by the Nix
+Daemon. The Nix Daemon cannot prompt for a passphrase via the terminal
+or `ssh-agent`, so the SSH key must not have a passphrase.
+
+In addition, the Nix Daemon's user (typically root) needs to have SSH
+access to the remote builder.
+
+Access can be verified by running `sudo su`, and then validating SSH
+access, e.g. by running `ssh mac`. SSH identity files for root users
+are usually stored in `/root/.ssh/` (Linux) or `/var/root/.ssh` (MacOS).
+
+If you get the error
+
+```console
+bash: nix: command not found
+error: cannot connect to 'mac'
+```
+
+then you need to ensure that the `PATH` of non-interactive login shells
+contains Nix.
+
+The [list of remote build machines](@docroot@/command-ref/conf-file.md#conf-builders) can be specified on the command line or in the Nix configuration file.
+For example, the following command allows you to build a derivation for `x86_64-darwin` on a Linux machine:
+
+```console
+uname
+```
+
+```console
+Linux
+```
+
+```console
+nix build --impure \
+ --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+ --builders 'ssh://mac x86_64-darwin'
+```
+
+```console
+[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
+```
+
+```console
+cat ./result
+```
+
+```console
+Darwin
+```
+
+It is possible to specify multiple build machines separated by a semicolon or a newline, e.g.
+
+```console
+  --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
+```
+
+Remote build machines can also be configured in [`nix.conf`](@docroot@/command-ref/conf-file.md), e.g.
+
+    builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd
+
+After making changes to `nix.conf`, restart the Nix daemon for changes to take effect.
+
+Finally, remote build machines can be configured in a separate configuration
+file included in `builders` via the syntax `@/path/to/file`. For example,
+
+    builders = @/etc/nix/machines
+
+causes the list of machines in `/etc/nix/machines` to be included.
+(This is the default.)

doc/manual/source/command-ref/env-common.md

@@ -138,6 +138,19 @@ The following environment variables are used to determine locations of various s
 - [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`)
 - [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`)
 
-
 [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
 [`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories
+
+In addition, setting the following environment variables overrides the XDG base directories:
+
+- [`NIX_CONFIG_HOME`]{#env-NIX_CONFIG_HOME} (default `$XDG_CONFIG_HOME/nix`)
+- [`NIX_STATE_HOME`]{#env-NIX_STATE_HOME} (default `$XDG_STATE_HOME/nix`)
+- [`NIX_CACHE_HOME`]{#env-NIX_CACHE_HOME} (default `$XDG_CACHE_HOME/nix`)
+
+When [`use-xdg-base-directories`] is enabled, the configuration directory is:
+
+1. `$NIX_CONFIG_HOME`, if it is defined
+2. Otherwise, `$XDG_CONFIG_HOME/nix`, if `XDG_CONFIG_HOME` is defined
+3. Otherwise, `~/.config/nix`.
+
+Likewise for the state and cache directories.

doc/manual/source/command-ref/files/default-nix-expression.md

@@ -1,6 +1,6 @@
 ## Default Nix expression
 
-The source for the default [Nix expressions](@docroot@/language/index.md) used by [`nix-env`]:
+The source for the [Nix expressions](@docroot@/glossary.md#gloss-nix-expression) used by [`nix-env`] by default:
 
 - `~/.nix-defexpr`
 - `$XDG_STATE_HOME/nix/defexpr` if [`use-xdg-base-directories`] is set to `true`.
@@ -18,24 +18,25 @@ Then, the resulting expression is interpreted like this:
 - If the expression is an attribute set, it is used as the default Nix expression.
 - If the expression is a function, an empty set is passed as argument and the return value is used as the default Nix expression.
 
-
+> **Example**
-For example, if the default expression contains two files, `foo.nix` and `bar.nix`, then the default Nix expression will be equivalent to
+>
-
+> If the default expression contains two files, `foo.nix` and `bar.nix`, then the default Nix expression will be equivalent to
-```nix
+>
-{
+> ```nix
-  foo = import ~/.nix-defexpr/foo.nix;
+> {
-  bar = import ~/.nix-defexpr/bar.nix;
+>   foo = import ~/.nix-defexpr/foo.nix;
-}
+>   bar = import ~/.nix-defexpr/bar.nix;
-```
+> }
+> ```
 
 The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored.
 
-The command [`nix-channel`] places a symlink to the user's current [channels profile](@docroot@/command-ref/files/channels.md) in this directory.
+The command [`nix-channel`] places a symlink to the current user's [channels] in this directory, the [user channel link](#user-channel-link).
 This makes all subscribed channels available as attributes in the default expression.
 
 ## User channel link
 
-A symlink that ensures that [`nix-env`] can find your channels:
+A symlink that ensures that [`nix-env`] can find the current user's [channels]:
 
 - `~/.nix-defexpr/channels`
 - `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`.
@@ -45,8 +46,9 @@ This symlink points to:
 - `$XDG_STATE_HOME/profiles/channels` for regular users
 - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root`
 
-In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.[`nix-env`]: ../nix-env.md
+In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.
 
-[`nix-env`]: @docroot@/command-ref/nix-env.md
 [`nix-channel`]: @docroot@/command-ref/nix-channel.md
+[`nix-env`]: @docroot@/command-ref/nix-env.md
 [`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories
+[channels]: @docroot@/command-ref/files/channels.md

doc/manual/source/command-ref/meson.build

@@ -0,0 +1,63 @@
+xp_features_json = custom_target(
+  command : [nix, '__dump-xp-features'],
+  capture : true,
+  output : 'xp-features.json',
+)
+
+experimental_features_shortlist_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile ./@INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features-shortlist.nix',
+    xp_features_json,
+  ],
+  output : 'experimental-features-shortlist.md',
+  capture : true,
+  env : nix_env_for_docs,
+)
+
+nix3_cli_files = custom_target(
+  command : [
+    python.full_path(),
+    '@INPUT0@',
+    '@OUTPUT@',
+    '--'
+  ] + nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT1@ true (builtins.readFile ./@INPUT2@)',
+  ],
+  input : [
+    '../../remove_before_wrapper.py',
+    '../../generate-manpage.nix',
+    nix3_cli_json,
+  ],
+  output : 'new-cli',
+  env : nix_env_for_docs,
+)
+
+conf_file_md_body = custom_target(
+  command : [
+    nix_eval_for_docs,
+    '--expr',
+    'import @INPUT0@ { prefix = "conf"; } (builtins.fromJSON (builtins.readFile ./@INPUT1@))',
+  ],
+  capture : true,
+  input : [
+    '../../generate-settings.nix',
+    conf_file_json,
+  ],
+  output : 'conf-file.body.md',
+  env : nix_env_for_docs,
+)
+
+conf_file_md = custom_target(
+  command : [ 'cat', '@INPUT0@', '@INPUT1@' ],
+  capture : true,
+  input : [
+    'conf-file-prefix.md',
+    conf_file_md_body,
+  ],
+  output : 'conf-file.md',
+)

doc/manual/source/command-ref/nix-env.md

@@ -62,7 +62,7 @@ These pages can be viewed offline:
 
 Several operations, such as [`nix-env --query`](./nix-env/query.md) and [`nix-env --install`](./nix-env/install.md), take a list of *arguments* that specify the packages on which to operate.
 
-Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-names):
+Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-name):
 
 - `name`: Everything up to but not including the first dash (`-`) that is *not* followed by a letter.
 - `version`: The rest, excluding the separating dash.

doc/manual/source/command-ref/nix-store/query.md

@@ -104,7 +104,7 @@ symlink.
 
   Prints a set of derivation files (`.drv`) which are supposed produce
   said paths when realized. Might print nothing, for example for source paths
-  or paths subsituted from a binary cache.
+  or paths substituted from a binary cache.
 
 - `--graph`
 
@@ -241,4 +241,3 @@ $ nix-store --query --roots $(which svn)
 /nix/var/nix/profiles/default-82-link
 /home/eelco/.local/state/nix/profiles/profile-97-link
 ```
-

doc/manual/source/command-ref/opt-common.md

@@ -1,3 +1,7 @@
+<!-- Some of the options documented here are hardcopied from
+     src/libcmd/common-eval-args.cc
+-->
+
 # Common Options
 
 Most Nix commands accept the following command-line options:
@@ -161,6 +165,14 @@ Most Nix commands accept the following command-line options:
   You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`.
   (Note that since the argument is a Nix string literal, you have to escape the quotes.)
 
+- <span id="opt-arg-from-file">[`--arg-from-file`](#opt-arg-from-file)</span> *name* *path*
+
+  Pass the contents of file *path* as the argument *name* to Nix functions.
+
+- <span id="opt-arg-from-stdin">[`--arg-from-stdin`](#opt-arg-from-stdin)</span> *name*
+
+  Pass the contents of stdin as the argument *name* to Nix functions.
+
 - <span id="opt-argstr">[`--argstr`](#opt-argstr)</span> *name* *value*
 
   This option is like `--arg`, only the value is not a Nix expression but a string.
@@ -179,6 +191,10 @@ Most Nix commands accept the following command-line options:
   attribute of the fourth element of the array in the `foo` attribute
   of the top-level expression.
 
+- <span id="opt-eval-store">[`--eval-store`](#opt-eval-store)</span> *store-url*
+
+  The [URL to the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. where to store derivations (`.drv` files) and inputs referenced by them.
+
 - <span id="opt-expr">[`--expr`](#opt-expr)</span> / `-E`
 
   Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions.
@@ -194,6 +210,10 @@ Most Nix commands accept the following command-line options:
 
   Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH).
 
+- <span id="opt-impure">[`--impure`](#opt-impure)</span>
+
+  Allow access to mutable paths and repositories.
+
 - <span id="opt-option">[`--option`](#opt-option)</span> *name* *value*
 
   Set the Nix configuration option *name* to *value*.