Merge remote-tracking branch 'nixos/master'

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.1.1
+        uses: zeebe-io/backport-action@v2.2.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -20,12 +20,12 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -62,10 +62,10 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -114,12 +114,12 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'

.github/workflows/labels.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'NixOS'
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: false

.gitignore

@@ -21,12 +21,16 @@ perl/Makefile.config
 /doc/manual/language.json
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
+/doc/manual/src/SUMMARY-rl-next.md
+/doc/manual/src/store/types/*
+!/doc/manual/src/store/types/index.md.in
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
 /doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 /doc/manual/src/language/builtin-constants.md
+/doc/manual/src/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh
@@ -41,18 +45,18 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
-/src/libexpr/tests/libnixexpr-tests
+/tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
-/src/libstore/tests/libnixstore-tests
+/tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
-/src/libutil/tests/libnixutil-tests
+/tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
 
-/src/nix/doc
+/src/nix/generated-doc
 
 # /src/nix-env/
 /src/nix-env/nix-env

Makefile

@@ -1,5 +1,7 @@
--include Makefile.config
+include mk/build-dir.mk
-clean-files += Makefile.config
+
+-include $(buildprefix)Makefile.config
+clean-files += $(buildprefix)Makefile.config
 
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
@@ -19,17 +21,17 @@ makefiles = \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
-  misc/upstart/local.mk \
+  misc/upstart/local.mk
-  doc/manual/local.mk \
-  doc/internal-api/local.mk
 endif
 
 ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
-UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data
 makefiles += \
-  src/libutil/tests/local.mk \
+  tests/unit/libutil/local.mk \
-  src/libstore/tests/local.mk \
+  tests/unit/libutil-support/local.mk \
-  src/libexpr/tests/local.mk
+  tests/unit/libstore/local.mk \
+  tests/unit/libstore-support/local.mk \
+  tests/unit/libexpr/local.mk \
+  tests/unit/libexpr-support/local.mk
 endif
 
 ifeq ($(ENABLE_TESTS), yes)
@@ -55,4 +57,11 @@ endif
 
 include mk/lib.mk
 
+# Must be included after `mk/lib.mk` so rules refer to variables defined
+# by the library. Rules are not "lazy" like variables, unfortunately.
+ifeq ($(ENABLE_BUILD), yes)
+$(eval $(call include-sub-makefile, doc/manual/local.mk))
+$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
+endif
+
 GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src

boehmgc-traceable_allocator-public.diff

@@ -0,0 +1,12 @@
+diff --git a/include/gc_allocator.h b/include/gc_allocator.h
+index 597c7f13..587286be 100644
+--- a/include/gc_allocator.h
++++ b/include/gc_allocator.h
+@@ -312,6 +312,7 @@ public:
+ 
+ template<>
+ class traceable_allocator<void> {
++public:
+   typedef size_t      size_type;
+   typedef ptrdiff_t   difference_type;
+   typedef void*       pointer;

configure.ac

@@ -282,6 +282,8 @@ case "$host_os" in
 esac
 AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
+# Optional dependencies for better normalizing file system data
+AC_CHECK_HEADERS[sys/xattr.h]
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)

doc/internal-api/doxygen.cfg.in

@@ -39,17 +39,21 @@ INPUT                  = \
   src/libcmd \
   src/libexpr \
   src/libexpr/flake \
-  src/libexpr/tests \
+  tests/unit/libexpr \
-  src/libexpr/tests/value \
+  tests/unit/libexpr/value \
+  tests/unit/libexpr/test \
+  tests/unit/libexpr/test/value \
   src/libexpr/value \
   src/libfetchers \
   src/libmain \
   src/libstore \
   src/libstore/build \
   src/libstore/builtins \
-  src/libstore/tests \
+  tests/unit/libstore \
+  tests/unit/libstore/test \
   src/libutil \
-  src/libutil/tests \
+  tests/unit/libutil \
+  tests/unit/libutil/test \
   src/nix \
   src/nix-env \
   src/nix-store

doc/manual/_redirects

@@ -0,0 +1,40 @@
+# redirect rules for paths (server-side) to prevent link rot.
+# see ./redirects.js for redirects based on URL fragments (client-side)
+#
+# concrete user story this supports:
+# - user finds URL to the manual for Nix x.y
+# - Nix x.z (z > y) is the most recent release
+# - updating the version in the URL will show the right thing
+#
+# format documentation:
+# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
+# - https://docs.netlify.com/routing/redirects/redirect-options/
+#
+# conventions:
+# - always force (<CODE>!) since this allows re-using file names
+# - group related paths to ease readability
+# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts
+# - redirects that should have been there but are missing can be inserted where they belong
+
+/advanced-topics/advanced-topics /advanced-topics 301!
+
+/command-ref/command-ref /command-ref 301!
+
+/contributing/contributing /contributing 301!
+
+/expressions/expression-language /language/ 301!
+/expressions/language-constructs /language/constructs 301!
+/expressions/language-operators /language/operators 301!
+/expressions/language-values /language/values 301!
+/expressions/* /language/:splat 301!
+
+/installation/installation /installation 301!
+
+/package-management/basic-package-mgmt /command-ref/nix-env 301!
+/package-management/channels* /command-ref/nix-channel 301!
+/package-management/package-management /package-management 301!
+/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+
+/protocols/protocols /protocols 301!
+
+/release-notes/release-notes /release-notes 301!

doc/manual/generate-builtin-constants.nix

@@ -1,6 +1,6 @@
 let
   inherit (builtins) concatStringsSep attrValues mapAttrs;
-  inherit (import ./utils.nix) optionalString squash;
+  inherit (import <nix/utils.nix>) optionalString squash;
 in
 
 builtinsInfo:

doc/manual/generate-builtins.nix

@@ -1,6 +1,6 @@
 let
   inherit (builtins) concatStringsSep attrValues mapAttrs;
-  inherit (import ./utils.nix) optionalString squash;
+  inherit (import <nix/utils.nix>) optionalString squash;
 in
 
 builtinsInfo:

doc/manual/generate-manpage.nix

@@ -1,9 +1,29 @@
 let
   inherit (builtins)
-    attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
+    attrNames
-    concatStringsSep concatMap length lessThan replaceStrings sort;
+    attrValues
-  inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
+    concatMap
-  showStoreDocs = import ./generate-store-info.nix;
+    concatStringsSep
+    fromJSON
+    groupBy
+    length
+    lessThan
+    listToAttrs
+    mapAttrs
+    match
+    replaceStrings
+    sort
+    ;
+  inherit (import <nix/utils.nix>)
+    attrsToList
+    concatStrings
+    filterAttrs
+    optionalString
+    squash
+    trim
+    unique
+    ;
+  showStoreDocs = import <nix/generate-store-info.nix>;
 in
 
 inlineHTML: commandDump:
@@ -31,7 +51,7 @@ let
 
         ${maybeSubcommands}
 
-        ${maybeStoreDocs}
+        ${maybeProse}
 
         ${maybeOptions}
       '';
@@ -71,25 +91,56 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      # FIXME: this is a hack.
+      maybeProse =
-      # store parameters should not be part of command documentation to begin
+        # FIXME: this is a horrible hack to keep `nix help-stores` working.
-      # with, but instead be rendered on separate pages.
+        # the correct answer to this is to remove that command and replace it
-      maybeStoreDocs = optionalString (details ? doc)
+        # by statically generated manpages or the output of something like `nix
-        (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc);
+        # store info <store type>`.
+        let
+          help-stores = ''
+            ${index}
 
-      maybeOptions = let
+            ${allStores}
-        allVisibleOptions = filterAttrs
+          '';
-          (_: o: ! o.hiddenCategory)
+          index = replaceStrings
-          (details.flags // toplevel.flags);
+            [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ]
-      in optionalString (allVisibleOptions != {}) ''
+            [ storesOverview "#local-store" "#local-daemon-store" ]
-        # Options
+            details.doc;
+          storesOverview =
+            let
+              showEntry = store:
+                "- [${store.name}](#${store.slug})";
+            in
+            concatStringsSep "\n" (map showEntry storesList) + "\n";
+          allStores = concatStringsSep "\n" (attrValues storePages);
+          storePages = listToAttrs
+            (map (s: { name = s.filename; value = s.page; }) storesList);
+          storesList = showStoreDocs {
+            storeInfo = commandInfo.stores;
+            inherit inlineHTML;
+          };
+        in
+        optionalString (details ? doc) (
+          if match "@store-types@" details.doc != [ ]
+          then help-stores
+          else details.doc
+        );
 
-        ${showOptions inlineHTML allVisibleOptions}
+      maybeOptions =
+        let
+          allVisibleOptions = filterAttrs
+            (_: o: ! o.hiddenCategory)
+            (details.flags // toplevel.flags);
+        in
+        optionalString (allVisibleOptions != { }) ''
+          # Options
 
-        > **Note**
+          ${showOptions inlineHTML allVisibleOptions}
-        >
+
-        > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
+          > **Note**
-      '';
+          >
+          > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
+        '';
 
       showOptions = inlineHTML: allOptions:
         let
@@ -97,7 +148,7 @@ let
             ${optionalString (cat != "") "## ${cat}"}
 
             ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))}
-            '';
+          '';
           showOption = name: option:
             let
               result = trim ''

doc/manual/generate-settings.nix

@@ -1,6 +1,6 @@
 let
   inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs;
-  inherit (import ./utils.nix) concatStrings indent optionalString squash;
+  inherit (import <nix/utils.nix>) concatStrings indent optionalString squash;
 in
 
 # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown`
@@ -31,18 +31,19 @@ let
 
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
           > **Warning**
+          >
           > This setting is part of an
           > [experimental feature](@docroot@/contributing/experimental-features.md).
-
+          >
-          To change this setting, you need to make sure the corresponding experimental feature,
+          > To change this setting, make sure the
-          [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
+          > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature})
-          is enabled.
+          > is enabled.
-          For example, include the following in [`nix.conf`](#):
+          > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
-
+          >
-          ```
+          > ```
-          extra-experimental-features = ${experimentalFeature}
+          > extra-experimental-features = ${experimentalFeature}
-          ${setting} = ...
+          > ${setting} = ...
-          ```
+          > ```
         '';
 
       showDefault = documentDefault: defaultValue:

doc/manual/generate-store-info.nix

@@ -1,45 +1,57 @@
 let
-  inherit (builtins) attrValues mapAttrs;
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
-  inherit (import ./utils.nix) concatStrings optionalString;
+  inherit (import <nix/utils.nix>) optionalString filterAttrs trim squash toLower unique indent;
-  showSettings = import ./generate-settings.nix;
+  showSettings = import <nix/generate-settings.nix>;
 in
 
-inlineHTML: storesInfo:
+{
+  # data structure describing all stores and their parameters
+  storeInfo,
+  # whether to add inline HTML tags
+  # `lowdown` does not eat those for one of the output modes
+  inlineHTML,
+}:
 
 let
 
-  showStore = name: { settings, doc, experimentalFeature }:
+  showStore = { name, slug }: { settings, doc, experimentalFeature }:
     let
+      result = squash ''
+        # ${name}
 
-    result = ''
+        ${doc}
-      ## ${name}
 
-      ${doc}
+        ${experimentalFeatureNote}
 
-      ${experimentalFeatureNote}
+        ## Settings
 
-      ### Settings
+        ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
+      '';
 
-      ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
+      experimentalFeatureNote = optionalString (experimentalFeature != null) ''
-    '';
+        > **Warning**
+        >
+        > This store is part of an
+        > [experimental feature](@docroot@/contributing/experimental-features.md).
+        >
+        > To use this store, make sure the
+        > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature})
+        > is enabled.
+        > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+        >
+        > ```
+        > extra-experimental-features = ${experimentalFeature}
+        > ```
+      '';
+    in result;
 
-      # markdown doesn't like spaces in URLs
+  storesList = map
-      slug = builtins.replaceStrings [ " " ] [ "-" ] name;
+    (name: rec {
+      inherit name;
+      slug = replaceStrings [ " " ] [ "-" ] (toLower name);
+      filename = "${slug}.md";
+      page = showStore { inherit name slug; } storeInfo.${name};
+    })
+    (attrNames storeInfo);
 
-    experimentalFeatureNote = optionalString (experimentalFeature != null) ''
+in storesList
-      > **Warning**
-      > This store is part of an
-      > [experimental feature](@docroot@/contributing/experimental-features.md).
-
-      To use this store, you need to make sure the corresponding experimental feature,
-      [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
-      is enabled.
-      For example, include the following in [`nix.conf`](#):
-
-      ```
-      extra-experimental-features = ${experimentalFeature}
-      ```
-    '';
-  in result;
-
-in concatStrings (attrValues (mapAttrs showStore storesInfo))

doc/manual/generate-store-types.nix

@@ -0,0 +1,39 @@
+let
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  showSettings = import <nix/generate-settings.nix>;
+  showStoreDocs = import <nix/generate-store-info.nix>;
+in
+
+storeInfo:
+
+let
+  storesList = showStoreDocs {
+    inherit storeInfo;
+    inlineHTML = true;
+  };
+
+  index =
+    let
+      showEntry = store:
+        "- [${store.name}](./${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList);
+
+  "index.md" = replaceStrings
+    [ "@store-types@" ] [ index ]
+    (readFile ./src/store/types/index.md.in);
+
+  tableOfContents =
+    let
+      showEntry = store:
+        "    - [${store.name}](store/types/${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList) + "\n";
+
+  "SUMMARY.md" = tableOfContents;
+
+  storePages = listToAttrs
+    (map (s: { name = s.filename; value = s.page; }) storesList);
+
+in
+storePages // { inherit "index.md" "SUMMARY.md"; }

doc/manual/generate-xp-features-shortlist.nix

@@ -1,5 +1,5 @@
 with builtins;
-with import ./utils.nix;
+with import <nix/utils.nix>;
 
 let
   showExperimentalFeature = name: doc:

doc/manual/generate-xp-features.nix

@@ -1,5 +1,5 @@
 with builtins;
-with import ./utils.nix;
+with import <nix/utils.nix>;
 
 let
   showExperimentalFeature = name: doc:
@@ -8,4 +8,6 @@ let
 
       ${doc}
     '';
-in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))
+in
+
+xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/local.mk

@@ -1,5 +1,10 @@
 ifeq ($(doc_generate),yes)
 
+# The version of Nix used to generate the doc. Can also be
+# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`),
+# if one prefers.
+doc_nix = $(nix_PATH)
+
 MANUAL_SRCS := \
 	$(call rwildcard, $(d)/src, *.md) \
 	$(call rwildcard, $(d)/src, */*.md)
@@ -24,7 +29,7 @@ man-pages += $(foreach subcommand, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
-# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
+# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
@@ -32,7 +37,7 @@ dummy-env = env -i \
 	NIX_STATE_DIR=/dummy \
 	NIX_CONFIG='cores = 0'
 
-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
+nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
 
 # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
 define process-includes
@@ -92,58 +97,83 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix
+$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix)
+	@# FIXME: build out of tree!
+	@rm -rf $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores'
+	@# do not destroy existing contents
+	@mv $@.tmp/* $@/
+
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix)
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
-$(d)/nix.json: $(bindir)/nix
+$(d)/nix.json: $(doc_nix)
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/conf-file.json: $(bindir)/nix
+$(d)/conf-file.json: $(doc_nix)
-	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
+	$(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
+$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
+$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/xp-features.json: $(bindir)/nix
+$(d)/xp-features.json: $(doc_nix)
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
+$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix
+$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
 	@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/language.json: $(bindir)/nix
+$(d)/language.json: $(doc_nix)
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp
 	@mv $@.tmp $@
 
+# Generate "Upcoming release" notes (or clear it and remove from menu)
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
+	@if type -p changelog-d > /dev/null; then \
+		echo "  GEN   " $@; \
+		changelog-d doc/manual/rl-next > $@; \
+	else \
+		echo "  NULL  " $@; \
+		true > $@; \
+	fi
+
+$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+	$(trace-gen) true
+	@if [ -s $< ]; then \
+		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
+	else \
+	  true > $@; \
+	fi
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
@@ -177,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/redirects.js

@@ -1,7 +1,9 @@
-// redirect rules for anchors ensure backwards compatibility of URLs.
+// redirect rules for URL fragments (client-side) to prevent link rot.
-// this must be done on the client side, as web servers do not see the anchor part of the URL.
+// this must be done on the client side, as web servers do not see the fragment part of the URL.
+// it will only work with JavaScript enabled in the browser, but this is the best we can do here.
+// see ./_redirects for path redirects (client-side)
 
-// redirections are declared as follows:
+// redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
 //
 //     IMPORTANT: it must specify the full path with file name and suffix

doc/manual/rl-next/config

@@ -0,0 +1,2 @@
+organization: NixOS
+repository: nix

doc/manual/rl-next/mounted-ssh-store.md

@@ -0,0 +1,9 @@
+synopsis: Mounted SSH Store
+issues: #7890
+prs: #7912
+description: {
+
+Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
+
+}

doc/manual/rl-next/nix-config-show.md

@@ -0,0 +1,8 @@
+synopsis: `nix config show`
+issues: #7672
+prs: #9477
+description: {
+
+`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface.
+
+}

doc/manual/rl-next/nix-env-json-drv-path.md

@@ -0,0 +1,9 @@
+synopsis: Fix `nix-env --query --drv-path --json`
+prs: #9257
+description: {
+
+Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+}
+
+

doc/manual/src/SUMMARY.md.in

@@ -2,7 +2,7 @@
 
 - [Introduction](introduction.md)
 - [Quick Start](quick-start.md)
-- [Installation](installation/installation.md)
+- [Installation](installation/index.md)
   - [Supported Platforms](installation/supported-platforms.md)
   - [Installing a Binary Distribution](installation/installing-binary.md)
   - [Installing Nix from Source](installation/installing-source.md)
@@ -20,6 +20,8 @@
   - [File System Object](store/file-system-object.md)
   - [Store Object](store/store-object.md)
   - [Store Path](store/store-path.md)
+  - [Store Types](store/types/index.md)
+{{#include ./store/types/SUMMARY.md}}
 - [Nix Language](language/index.md)
   - [Data Types](language/values.md)
   - [Language Constructs](language/constructs.md)
@@ -31,11 +33,11 @@
     - [Import From Derivation](language/import-from-derivation.md)
   - [Built-in Constants](language/builtin-constants.md)
   - [Built-in Functions](language/builtins.md)
-- [Package Management](package-management/package-management.md)
+- [Package Management](package-management/index.md)
   - [Profiles](package-management/profiles.md)
   - [Garbage Collection](package-management/garbage-collection.md)
     - [Garbage Collector Roots](package-management/garbage-collector-roots.md)
-- [Advanced Topics](advanced-topics/advanced-topics.md)
+- [Advanced Topics](advanced-topics/index.md)
   - [Sharing Packages Between Machines](package-management/sharing-packages.md)
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
     - [Copying Closures via SSH](package-management/copy-closure.md)
@@ -45,7 +47,7 @@
   - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
   - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
   - [Using the `post-build-hook`](advanced-topics/post-build-hook.md)
-- [Command Reference](command-ref/command-ref.md)
+- [Command Reference](command-ref/index.md)
   - [Common Options](command-ref/opt-common.md)
   - [Common Environment Variables](command-ref/env-common.md)
   - [Main Commands](command-ref/main-commands.md)
@@ -102,19 +104,19 @@
     - [Channels](command-ref/files/channels.md)
     - [Default Nix expression](command-ref/files/default-nix-expression.md)
 - [Architecture and Design](architecture/architecture.md)
-- [Protocols](protocols/protocols.md)
+- [Protocols](protocols/index.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
 - [Glossary](glossary.md)
-- [Contributing](contributing/contributing.md)
+- [Contributing](contributing/index.md)
   - [Hacking](contributing/hacking.md)
   - [Testing](contributing/testing.md)
   - [Documentation](contributing/documentation.md)
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
-- [Release Notes](release-notes/release-notes.md)
+- [Release Notes](release-notes/index.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+{{#include ./SUMMARY-rl-next.md}}
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)

doc/manual/src/architecture/architecture.md

@@ -52,7 +52,7 @@ The following [concept map] shows its main components (rectangles), the objects
                                             '---------------'
 ```
 
-At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers.
+At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers.
 
 The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*.
 

doc/manual/src/contributing/cli-guideline.md

@@ -87,7 +87,7 @@ impacted the most by bad user experience.
       and [aligning of text](#text-alignment).
     - [Autocomplete](#shell-completion) of options.
 
-  Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ...
+  Examples of such commands: `nix edit`, `nix eval`, ...
 
 - **Utility and scripting commands**
 
@@ -426,7 +426,7 @@ This leads to the following guidelines:
 ### Examples
 
 
-This is bad, because all keys must be assumed to be store implementations:
+This is bad, because all keys must be assumed to be store types:
 
 ```json
 {

doc/manual/src/contributing/hacking.md

@@ -10,7 +10,7 @@ $ cd nix
 
 The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions].
 
-[installation instructions]: ../installation/installation.md
+[installation instructions]: ../installation/index.md
 
 ## Building Nix with flakes
 
@@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default
 Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
 Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
 
+### Building for multiple platforms at once
+
+It is useful to perform multiple cross and native builds on the same source tree,
+for example to ensure that better support for one platform doesn't break the build for another.
+In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code:
+
+1. Create a directory for the build, e.g.
+
+   ```bash
+   mkdir build
+   ```
+
+2. Run the configure script from that directory, e.g.
+
+   ```bash
+   cd build
+   ../configure <configure flags>
+   ```
+
+3. Run make from the source directory, but with the build directory specified, e.g.
+
+   ```bash
+   make builddir=build <make flags>
+   ```
+
 ## System type
 
 Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
@@ -220,3 +245,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`.
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
+
+## Add a release note
+
+`doc/manual/rl-next` contains release notes entries for all unreleased changes.
+
+User-visible changes should come with a release note.
+
+### Add an entry
+
+Here's what a complete entry looks like. The file name is not incorporated in the document.
+
+```
+synopsis: Basically a title
+issues: #1234
+prs: #1238
+description: {
+
+Here's one or more paragraphs that describe the change.
+
+- It's markdown
+- Add references to the manual using @docroot@
+
+}
+```
+
+Significant changes should add the following header, which moves them to the top.
+
+```
+significance: significant
+```
+
+<!-- Keep an eye on https://codeberg.org/fgaz/changelog-d/issues/1 -->
+See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog).
+
+### Build process
+
+Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
+Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.

doc/manual/src/contributing/testing.md

@@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
 
 [googletest]: https://google.github.io/googletest/
 [rapidcheck]: https://github.com/emil-e/rapidcheck
+[property testing]: https://en.wikipedia.org/wiki/Property_testing
 
 ### Source and header layout
 
@@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
 > ```
 > src
 > ├── libexpr
+> │   ├── local.mk
 > │   ├── value/context.hh
 > │   ├── value/context.cc
+> │   …
+> │
+> ├── tests
 > │   │
 > │   …
->     └── tests
+> │   └── unit
-> │       ├── value/context.hh
+> │       ├── libutil
-> │       ├── value/context.cc
+> │       │   ├── local.mk
+> │       │   …
+> │       │   └── data
+> │       │       ├── git/tree.txt
+> │       │       …
 > │       │
-> │       …
+> │       ├── libexpr-support
-> │
+> │       │   ├── local.mk
-> ├── unit-test-data
+> │       │   └── tests
-> │   ├── libstore
+> │       │       ├── value/context.hh
-> │   │   ├── worker-protocol/content-address.bin
+> │       │       ├── value/context.cc
-> │   │   …
+> │       │       …
-> │   …
+> │       │
+> │       ├── libexpr
+> │       …   ├── local.mk
+> │           ├── value/context.cc
+> │           …
 > …
 > ```
 
-The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`).
+The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`.
+Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`.
 
-The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes.
+Data for unit tests is stored in a `data` subdir of the directory for each unit test executable.
-For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`.
+For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`.
-The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+Note that each executable only gets the data for its tests.
 
-> **Note**
+The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`.
-> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests.
+All headers are in a `tests` subdirectory so they are included with `#include "tests/"`.
-> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library.
+
-> That is why we have the test data nested within a single `unit-test-data` directory.
+The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing.
+But organizing the tests this way has one big benefit:
+there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library.
 
 ### Running tests
 
@@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional)
 Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used.
 For example:
 ```shell-session
-$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN
+$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN
 ...
 [  SKIPPED ] WorkerProtoTest.string_read
 [  SKIPPED ] WorkerProtoTest.string_write
@@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN
 will regenerate the "golden master" expected result for the `libnixstore` characterisation tests.
 The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything.
 
+### Unit test support libraries
+
+There are headers and code which are not just used to test the library in question, but also downstream libraries.
+For example, we do [property testing] with the [rapidcheck] library.
+This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests.
+Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it.
+Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them.
+
+It is important that these testing libraries don't contain any actual tests themselves.
+On some platforms they would be run as part of every test executable that uses them, which is redundant.
+On other platforms they wouldn't be run at all.
+
 ## Functional tests
 
 The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.

doc/manual/src/language/values.md

@@ -116,6 +116,7 @@
   [store path]: ../glossary.md#gloss-store-path
 
   Paths can include [string interpolation] and can themselves be [interpolated in other expressions].
+
   [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions
 
   At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.

doc/manual/src/quick-start.md

@@ -13,7 +13,7 @@ to subsequent chapters.
    The install script will use `sudo`, so make sure you have sufficient rights.
    On Linux, `--daemon` can be omitted for a single-user install.
 
-   For other installation methods, see [here](installation/installation.md).
+   For other installation methods, see [here](installation/index.md).
 
 1. See what installable packages are currently available in the
    channel:

doc/manual/src/release-notes/rl-2.19.md

@@ -18,7 +18,7 @@
 - `nix-shell` shebang lines now support single-quoted arguments.
 
 - `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
-  As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes.
+  This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
 
 - The interface for creating and updating lock files has been overhauled:
 

doc/manual/src/release-notes/rl-next.md

@@ -1,6 +0,0 @@
-# Release X.Y (202?-??-??)
-
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
-  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

doc/manual/src/store/index.md

@@ -2,4 +2,4 @@
 
 The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data.
 
-There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches.
+There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md).

doc/manual/src/store/types/index.md.in

@@ -1,6 +1,6 @@
-R"(
+Nix supports different types of stores:
 
-Nix supports different types of stores. These are described below.
+@store-types@
 
 ## Store URL format
 
@@ -29,18 +29,15 @@ supported settings for each store type are documented below.
 The special store URL `auto` causes Nix to automatically select a
 store as follows:
 
-* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix`
+* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix`
   is writable by the current user.
 
 * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect
-  to the Nix daemon listening on that socket](#local-daemon-store).
+  to the Nix daemon listening on that socket](./local-daemon-store.md).
 
-* Otherwise, on Linux only, use the [local chroot store](#local-store)
+* Otherwise, on Linux only, use the [local chroot store](./local-store.md)
   `~/.local/share/nix/root`, which will be created automatically if it
   does not exist.
 
-* Otherwise, use the [local store](#local-store) `/nix/store`.
+* Otherwise, use the [local store](./local-store.md) `/nix/store`.
 
-@stores@
-
-)"

doc/manual/utils.nix

@@ -1,5 +1,11 @@
 with builtins;
 
+let
+  lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz";
+  upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  stringToCharacters = s: genList (p: substring p 1 s) (stringLength s);
+in
+
 rec {
   splitLines = s: filter (x: !isList x) (split "\n" s);
 
@@ -18,6 +24,8 @@ rec {
     in
       if replaced == string then string else replaceStringsRec from to replaced;
 
+  toLower = replaceStrings upperChars lowerChars;
+
   squash = replaceStringsRec "\n\n\n" "\n\n";
 
   trim = string:

flake.lock

@@ -50,11 +50,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1700342017,
+        "lastModified": 1700748986,
-        "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=",
+        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "decdf666c833a325cb4417041a90681499e06a41",
+        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
         "type": "github"
       },
       "original": {

flake.nix

@@ -14,6 +14,9 @@
 
       officialRelease = false;
 
+      # Set to true to build the release notes for the next release.
+      buildUnreleasedNotes = false;
+
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
@@ -90,7 +93,7 @@
           ./misc
           ./precompiled-headers.h
           ./src
-          ./unit-test-data
+          ./tests/unit
           ./COPYING
           ./scripts/local.mk
           functionalTestFiles
@@ -173,6 +176,8 @@
           "--enable-internal-api-docs"
         ];
 
+        changelog-d = pkgs.buildPackages.changelog-d;
+
         nativeBuildDeps =
           [
             buildPackages.bison
@@ -190,7 +195,10 @@
             buildPackages.jq # Also for custom mdBook preprocessor.
             buildPackages.openssh # only needed for tests (ssh-keygen)
           ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
+          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
+          # Official releases don't have rl-next, so we don't need to compile a changelog
+          ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
+          ;
 
         buildDeps =
           [ curl
@@ -230,6 +238,9 @@
             }).overrideAttrs(o: {
               patches = (o.patches or []) ++ [
                 ./boehmgc-coroutine-sp-fallback.diff
+
+                # https://github.com/ivmai/bdwgc/pull/586
+                ./boehmgc-traceable_allocator-public.diff
               ];
             })
             )
@@ -727,6 +738,8 @@
               ++ lib.optional
                 (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
                 pkgs.buildPackages.clang-tools
+              # We want changelog-d in the shell even if the current build doesn't need it
+              ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
               ;
 
             buildInputs = buildDeps ++ propagatedDeps

maintainers/release-notes

@@ -0,0 +1,179 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small
+#                                   ^^^^^^^
+#                Only used for bash. shell.nix goes to the flake.
+
+# --- CONFIGURATION ---
+
+# This does double duty for
+# - including rl-next
+# - marking where to insert new links (right after)
+SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}'
+
+# --- LIB ---
+
+log() {
+  echo 1>&2 "release-notes:" "$@"
+}
+logcmd() {
+  local cmd="$1"
+  shift
+  logcmd2 "$cmd" "${*@Q}" "$cmd" "$@"
+}
+logcmd2() {
+  local fakecmd="$1"
+  local fakeargs="$2"
+  shift
+  shift
+  printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m "
+  echo "$fakeargs" 1>&2
+  "$@"
+}
+die() {
+  # ANSI red
+  printf 1>&2 "release-notes: \033[31;1merror:\033[0m"
+  echo 1>&2 "" "$@"
+  exit 1
+}
+confirm() {
+  local answer
+  echo 1>&2 "$@" "[y/n]"
+  read -r answer
+  case "$answer" in
+    y|Y|yes|Yes|YES)
+      return 0
+      ;;
+    n|N|no|No|NO)
+      return 1
+      ;;
+    *)
+      echo 1>&2 "please answer y or n"
+      confirm "$@"
+      ;;
+  esac
+}
+report_done() {
+  logcmd2 "git" "show" git -c pager.show=false show
+  printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n"
+}
+
+# --- PARSE ARGS ---
+
+if [[ $# -gt 0 ]]; then
+  die "Release notes takes no arguments, but make sure to set VERSION."
+fi
+
+# --- CHECKS ---
+
+if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then
+  die "must run in repo root"
+  exit 1
+fi
+
+# repo must be clean
+if ! git diff --quiet; then
+  die "repo is dirty, please commit or stash changes"
+fi
+
+if ! git diff --quiet --cached; then
+  die "repo has staged changes, please commit or stash them"
+fi
+
+if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then
+  # would have been nice to catch this early, but won't be worth the extra infra
+  die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script."
+fi
+
+if [[ ! -n "${VERSION:-}" ]]; then
+  die "please set the VERSION environment variable before invoking this script"
+  exit 1
+fi
+
+# version_major_minor:      MAJOR.MINOR
+# version_full:             MAJOR.MINOR.PATCH
+# IS_PATCH:                 true if this is a patch release; append instead of create
+if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is minor'
+  IS_PATCH=false
+  version_full="$VERSION.0"
+  version_major_minor="$VERSION"
+elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then
+  log 'is minor (.0)'
+  IS_PATCH=false
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')"
+elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is patch'
+  IS_PATCH=true
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
+else
+  die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)"
+fi
+
+unset VERSION
+
+log "version_major_minor=$version_major_minor"
+log "version_full=$version_full"
+log "IS_PATCH=$IS_PATCH"
+
+basename=rl-${version_major_minor}.md
+file=doc/manual/src/release-notes/$basename
+
+if ! $IS_PATCH; then
+  if [[ -e $file ]]; then
+    die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1"
+  fi
+fi
+
+# --- DEFAULTS ---
+
+if [[ ! -n "${DATE:-}" ]]; then
+  DATE="$(date +%Y-%m-%d)"
+  log "DATE not set, using $DATE"
+fi
+
+case "$DATE" in
+  [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])
+    ;;
+  *)
+    die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"
+    ;;
+esac
+
+# --- DO THE WORK ---
+
+# menu
+title="Release $version_major_minor ($DATE)"
+# section on page
+section_title="Release $version_full ($DATE)"
+
+(
+  # TODO add minor number, and append?
+  echo "# $section_title"
+  echo
+  changelog-d doc/manual/rl-next | sed -e 's/ *$//'
+) | tee -a $file
+
+log "Wrote $file"
+
+if ! $IS_PATCH; then
+  NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
+
+  # find the marker line, insert new link after it
+  escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+fi
+
+for f in doc/manual/rl-next/*.md; do
+  if [[ config != "$(basename $f)" ]]; then
+    logcmd git rm $f
+  fi
+done
+
+logcmd git add $file doc/manual/src/SUMMARY.md.in
+logcmd git status
+logcmd git commit -m "release notes: $version_full"
+
+report_done

maintainers/release-process.md

@@ -24,34 +24,23 @@ release:
 * In a checkout of the Nix repo, make sure you're on `master` and run
   `git pull`.
 
-* Move the contents of `doc/manual/src/release-notes/rl-next.md`
+* Compile the release notes by running
-  (except the first line) to
-  `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is
-  the contents of `.version` *without* the patch level, e.g. `2.12`
-  rather than `2.12.0`).
-
-* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like
-
-  ```
-  # Release 2.12 (2022-12-06)
-  ```
-
-* Proof-read / edit / rearrange the release notes. Breaking changes
-  and highlights should go to the top.
-
-* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in`
-  (*not* `SUMMARY.md`), e.g.
-
-  ```
-  - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
-  ```
-
-* Run
 
   ```console
   $ git checkout -b release-notes
-  $ git add doc/manual/src/release-notes/rl-$VERSION.md
+  $ VERSION=X.YY ./maintainers/release-notes
-  $ git commit -a -m 'Release notes'
+  ```
+
+  where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.
+
+  A commit is created.
+
+* Proof-read / edit / rearrange the release notes if needed. Breaking changes
+  and highlights should go to the top.
+
+* Push.
+
+  ```console
   $ git push --set-upstream $REMOTE release-notes
   ```
 
@@ -67,15 +56,17 @@ release:
   $ git checkout -b $VERSION-maintenance
   ```
 
-* Mark the release as stable:
+* Mark the release as official:
 
   ```console
-  $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a
+  $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix
   ```
 
   This removes the link to `rl-next.md` from the manual and sets
   `officialRelease = true` in `flake.nix`.
 
+* Commit
+
 * Push the release branch:
 
   ```console
@@ -159,6 +150,30 @@ release:
 
 ## Creating a point release
 
+* Checkout.
+
+  ```console
+  $ git checkout XX.YY-maintenance
+  ```
+
+* Determine the next patch version.
+
+  ```console
+  $ export VERSION=XX.YY.ZZ
+  ```
+
+* Update release notes.
+
+  ```console
+  $ ./maintainers/release-notes
+  ```
+
+* Push.
+
+  ```console
+  $ git push
+  ```
+
 * Wait for the desired evaluation of the maintenance jobset to finish
   building.
 

mk/build-dir.mk

@@ -0,0 +1,10 @@
+# Initialise support for build directories.
+builddir ?=
+
+ifdef builddir
+  buildprefix = $(builddir)/
+  buildprefixrel = $(builddir)
+else
+  buildprefix =
+  buildprefixrel = .
+endif

mk/common-test.sh

@@ -1,7 +1,7 @@
 # Remove overall test dir (at most one of the two should match) and
 # remove file extension.
 test_name=$(echo -n "$test" | sed \
-    -e "s|^unit-test-data/||" \
+    -e "s|^tests/unit/[^/]*/data/||" \
     -e "s|^tests/functional/||" \
     -e "s|\.sh$||" \
     )

mk/install-dirs.mk

@@ -0,0 +1,11 @@
+# Default installation paths.
+prefix ?= /usr/local
+libdir ?= $(prefix)/lib
+bindir ?= $(prefix)/bin
+libexecdir ?= $(prefix)/libexec
+datadir ?= $(prefix)/share
+localstatedir ?= $(prefix)/var
+sysconfdir ?= $(prefix)/etc
+mandir ?= $(prefix)/share/man
+
+DESTDIR ?=

mk/lib.mk

@@ -43,27 +43,6 @@ define newline
 endef
 
 
-# Default installation paths.
-prefix ?= /usr/local
-libdir ?= $(prefix)/lib
-bindir ?= $(prefix)/bin
-libexecdir ?= $(prefix)/libexec
-datadir ?= $(prefix)/share
-localstatedir ?= $(prefix)/var
-sysconfdir ?= $(prefix)/etc
-mandir ?= $(prefix)/share/man
-
-
-# Initialise support for build directories.
-builddir ?=
-
-ifdef builddir
-  buildprefix = $(builddir)/
-else
-  buildprefix =
-endif
-
-
 # Pass -fPIC if we're building dynamic libraries.
 BUILD_SHARED_LIBS ?= 1
 
@@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1)
 endif
 
 
+include mk/build-dir.mk
+include mk/install-dirs.mk
 include mk/functions.mk
 include mk/tracing.mk
 include mk/clean.mk
@@ -112,7 +93,7 @@ define include-sub-makefile
   include $(1)
 endef
 
-$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf))))
+$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf))))
 
 
 # Instantiate stuff.

mk/programs.mk

@@ -87,6 +87,6 @@ define build-program
   # Phony target to run this program (typically as a dependency of 'check').
   .PHONY: $(1)_RUN
   $(1)_RUN: $$($(1)_PATH)
-	$(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH)
+	$(trace-test) $$($(1)_ENV) $$($(1)_PATH)
 
 endef

mk/templates.mk

@@ -10,10 +10,10 @@ endef
 
 ifneq ($(MAKECMDGOALS), clean)
 
-%.h: %.h.in
+$(buildprefix)%.h: %.h.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --header=$@
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%)
 
-%: %.in
+$(buildprefix)%: %.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --file=$@
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%)
 
 endif

perl/lib/Nix/Store.xs

@@ -9,6 +9,7 @@
 #undef do_close
 
 #include "derivations.hh"
+#include "realisation.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "crypto.hh"

src/libcmd/built-path.cc

@@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const
     );
 }
 
-nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
+nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res;
     res["drvPath"] = drvPath->toJSON(store);
@@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
     return res;
 }
 
-nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
+nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res;
     res["drvPath"] = drvPath->toJSON(store);
@@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
     return res;
 }
 
-nlohmann::json SingleBuiltPath::toJSON(const Store & store) const
+nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
 {
     return std::visit([&](const auto & buildable) {
         return buildable.toJSON(store);
     }, raw());
 }
 
-nlohmann::json BuiltPath::toJSON(const Store & store) const
+nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
 {
     return std::visit([&](const auto & buildable) {
         return buildable.toJSON(store);

src/libcmd/built-path.hh

@@ -14,9 +14,9 @@ struct SingleBuiltPathBuilt {
 
     SingleDerivedPathBuilt discardOutputPath() const;
 
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
-    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     DECLARE_CMP(SingleBuiltPathBuilt);
 };
@@ -41,8 +41,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
 
     SingleDerivedPath discardOutputPath() const;
 
-    static SingleBuiltPath parse(const Store & store, std::string_view);
+    static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 };
 
 static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
@@ -59,9 +59,9 @@ struct BuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
-    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     DECLARE_CMP(BuiltPathBuilt);
 };
@@ -89,7 +89,7 @@ struct BuiltPath : _BuiltPathRaw {
     StorePathSet outPaths() const;
     RealisedPath::Set toRealisedPaths(Store & store) const;
 
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;

src/libcmd/common-eval-args.cc

@@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs()
         .longName = "eval-store",
         .description =
           R"(
-            The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+            The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
             to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
           )",
         .category = category,

src/libcmd/installables.cc

@@ -51,6 +51,22 @@ MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
 
+    addFlag({
+        .longName = "recreate-lock-file",
+        .description = R"(
+    Recreate the flake's lock file from scratch.
+
+    > **DEPRECATED**
+    >
+    > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
+        )",
+        .category = category,
+        .handler = {[&]() {
+            lockFlags.recreateLockFile = true;
+            warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead.");
+        }}
+    });
+
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
@@ -67,8 +83,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "no-registries",
-        .description =
+        .description = R"(
-          "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
+    Don't allow lookups in the flake registries.
+
+    > **DEPRECATED**
+    >
+    > Use [`--no-use-registries`](#opt-no-use-registries) instead.
+        )",
         .category = category,
         .handler = {[&]() {
             lockFlags.useRegistries = false;
@@ -83,6 +104,26 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {&lockFlags.commitLockFile, true}
     });
 
+    addFlag({
+        .longName = "update-input",
+        .description = R"(
+    Update a specific flake input (ignoring its previous entry in the lock file).
+
+    > **DEPRECATED**
+    >
+    > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
+        )",
+        .category = category,
+        .labels = {"input-path"},
+        .handler = {[&](std::string s) {
+            warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version.");
+            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
+        }},
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
+        }}
+    });
+
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",

src/libcmd/local.mk

@@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/repl.cc

@@ -43,7 +43,6 @@ extern "C" {
 #include "finally.hh"
 #include "markdown.hh"
 #include "local-fs-store.hh"
-#include "progress-bar.hh"
 #include "print.hh"
 
 #if HAVE_BOEHMGC
@@ -262,13 +261,11 @@ void NixRepl::mainLoop()
     rl_set_list_possib_func(listPossibleCallback);
 #endif
 
-    /* Stop the progress bar because it interferes with the display of
-       the repl. */
-    stopProgressBar();
-
     std::string input;
 
     while (true) {
+        // Hide the progress bar while waiting for user input, so that it won't interfere.
+        logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
         if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
@@ -278,6 +275,7 @@ void NixRepl::mainLoop()
             logger->cout("");
             break;
         }
+        logger->resume();
         try {
             if (!removeWhitespace(input).empty() && !processLine(input)) return;
         } catch (ParseError & e) {

src/libexpr/eval-cache.cc

@@ -22,7 +22,7 @@ struct AttrDb
 {
     std::atomic_bool failed{false};
 
-    const Store & cfg;
+    const StoreDirConfig & cfg;
 
     struct State
     {
@@ -39,7 +39,7 @@ struct AttrDb
     SymbolTable & symbols;
 
     AttrDb(
-        const Store & cfg,
+        const StoreDirConfig & cfg,
         const Hash & fingerprint,
         SymbolTable & symbols)
         : cfg(cfg)
@@ -323,7 +323,7 @@ struct AttrDb
 };
 
 static std::shared_ptr<AttrDb> makeAttrDb(
-    const Store & cfg,
+    const StoreDirConfig & cfg,
     const Hash & fingerprint,
     SymbolTable & symbols)
 {

src/libexpr/eval.cc

@@ -16,6 +16,7 @@
 #include "fs-input-accessor.hh"
 #include "memory-input-accessor.hh"
 #include "signals.hh"
+#include "gc-small-vector.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -31,6 +32,7 @@
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
+#include <boost/container/small_vector.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -1712,7 +1714,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                Value * vArgs[arity];
+                Value * vArgs[maxPrimOpArity];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
@@ -1775,11 +1777,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    Value * vArgs[args.size()];
+    SmallValueVector<4> vArgs(args.size());
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs, v, pos);
+    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
 }
 
 
@@ -2018,8 +2020,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    Value values[es->size()];
+    // List of returned strings. References to these Values must NOT be persisted.
-    Value * vTmpP = values;
+    SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
+    Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
         Value & vTmp = *vTmpP++;

src/libexpr/gc-small-vector.hh

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <boost/container/small_vector.hpp>
+
+#if HAVE_BOEHMGC
+
+#include <gc/gc.h>
+#include <gc/gc_cpp.h>
+#include <gc/gc_allocator.h>
+
+#endif
+
+namespace nix {
+
+struct Value;
+
+/**
+ * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap.
+ */
+#if HAVE_BOEHMGC
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems, traceable_allocator<T>>;
+#else
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems>;
+#endif
+
+/**
+ * A vector of value pointers. See `SmallVector`.
+ */
+template <size_t nItems>
+using SmallValueVector = SmallVector<Value *, nItems>;
+
+/**
+ * A vector of values that must not be referenced after the vector is destroyed.
+ *
+ * See also `SmallValueVector`.
+ */
+template <size_t nItems>
+using SmallTemporaryValueVector = SmallVector<Value, nItems>;
+
+}

src/libexpr/local.mk

@@ -36,7 +36,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libexpr/value/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
@@ -45,8 +45,6 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
 
 $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
-$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
 
-$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
+$(buildprefix)src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =
-
-src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =

src/libexpr/primops.cc

@@ -4,6 +4,7 @@
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
+#include "gc-small-vector.hh"
 #include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"
@@ -2729,7 +2730,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    Value * res[args[1]->listSize()];
+    SmallValueVector<nonRecursiveStackReservation> res(args[1]->listSize());
     size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
@@ -3064,8 +3065,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    // FIXME: putting this on the stack is risky.
+    SmallValueVector<nonRecursiveStackReservation> vs(args[1]->listSize());
-    Value * vs[args[1]->listSize()];
     size_t k = 0;
 
     bool same = true;
@@ -3461,7 +3461,8 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    Value lists[nrLists];
+    // List of returned lists before concatenation. References to these Values must NOT be persisted.
+    SmallTemporaryValueVector<conservativeStackReservation> lists(nrLists);
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {
@@ -4399,7 +4400,7 @@ void EvalState::createBaseEnv()
     addConstant("__currentSystem", v, {
         .type = nString,
         .doc = R"(
-          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system).
 
           It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
 
@@ -4448,7 +4449,7 @@ void EvalState::createBaseEnv()
         .doc = R"(
           Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use.
 
-          This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md):
+          This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format):
 
           ```shell-session
           $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir

src/libexpr/primops/fetchClosure.cc

@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "store-api.hh"
+#include "realisation.hh"
 #include "make-content-addressed.hh"
 #include "url.hh"
 

src/libexpr/tests/local.mk

@@ -1,23 +0,0 @@
-check: libexpr-tests_RUN
-
-programs += libexpr-tests
-
-libexpr-tests_NAME := libnixexpr-tests
-
-libexpr-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libexpr-tests_INSTALL_DIR := $(checkbindir)
-else
-  libexpr-tests_INSTALL_DIR :=
-endif
-
-libexpr-tests_SOURCES := \
-    $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc)
-
-libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
-
-libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
-
-libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock

src/libexpr/value.hh

@@ -67,7 +67,6 @@ class Symbol;
 class PosIdx;
 struct Pos;
 class StorePath;
-class Store;
 class EvalState;
 class XMLWriter;
 

src/libmain/local.mk

@@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil
 
 libmain_ALLOW_UNDEFINED = 1
 
-$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))

src/libstore/builtins/buildenv.cc

@@ -1,4 +1,5 @@
 #include "buildenv.hh"
+#include "derivations.hh"
 
 #include <sys/stat.h>
 #include <sys/types.h>

src/libstore/builtins/buildenv.hh

@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "derivations.hh"
 #include "store-api.hh"
 
 namespace nix {

src/libstore/common-protocol-impl.hh

@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \
+    TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<CommonProto, T >::read(store, conn); \
     } \
-    TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
+    TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
     }

src/libstore/common-protocol.cc

@@ -13,40 +13,40 @@ namespace nix {
 
 /* protocol-agnostic definitions */
 
-std::string CommonProto::Serialise<std::string>::read(const Store & store, CommonProto::ReadConn conn)
+std::string CommonProto::Serialise<std::string>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return readString(conn.from);
 }
 
-void CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
+void CommonProto::Serialise<std::string>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str)
 {
     conn.to << str;
 }
 
 
-StorePath CommonProto::Serialise<StorePath>::read(const Store & store, CommonProto::ReadConn conn)
+StorePath CommonProto::Serialise<StorePath>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return store.parseStorePath(readString(conn.from));
 }
 
-void CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
+void CommonProto::Serialise<StorePath>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath)
 {
     conn.to << store.printStorePath(storePath);
 }
 
 
-ContentAddress CommonProto::Serialise<ContentAddress>::read(const Store & store, CommonProto::ReadConn conn)
+ContentAddress CommonProto::Serialise<ContentAddress>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return ContentAddress::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
+void CommonProto::Serialise<ContentAddress>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca)
 {
     conn.to << renderContentAddress(ca);
 }
 
 
-Realisation CommonProto::Serialise<Realisation>::read(const Store & store, CommonProto::ReadConn conn)
+Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     std::string rawInput = readString(conn.from);
     return Realisation::fromJSON(
@@ -55,41 +55,41 @@ Realisation CommonProto::Serialise<Realisation>::read(const Store & store, Commo
     );
 }
 
-void CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
+void CommonProto::Serialise<Realisation>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
 {
     conn.to << realisation.toJSON().dump();
 }
 
 
-DrvOutput CommonProto::Serialise<DrvOutput>::read(const Store & store, CommonProto::ReadConn conn)
+DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return DrvOutput::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
+void CommonProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
 {
     conn.to << drvOutput.to_string();
 }
 
 
-std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const Store & store, CommonProto::ReadConn conn)
+std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     auto s = readString(conn.from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
+void CommonProto::Serialise<std::optional<StorePath>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
 {
     conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
-std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, CommonProto::ReadConn conn)
+std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return ContentAddress::parseOpt(readString(conn.from));
 }
 
-void CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
+void CommonProto::Serialise<std::optional<ContentAddress>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
 {
     conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
 }

src/libstore/common-protocol.hh

@@ -5,7 +5,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialized
@@ -48,7 +48,7 @@ struct CommonProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store, conn, t);
     }
@@ -57,8 +57,8 @@ struct CommonProto
 #define DECLARE_COMMON_SERIALISER(T) \
     struct CommonProto::Serialise< T > \
     { \
-        static T read(const Store & store, CommonProto::ReadConn conn); \
+        static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \
-        static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \
+        static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \
     }
 
 template<>

src/libstore/derivations.cc

@@ -11,7 +11,7 @@
 
 namespace nix {
 
-std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
+std::optional<StorePath> DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     return std::visit(overloaded {
         [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@@ -35,7 +35,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
 }
 
 
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
+StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     return store.makeFixedOutputPathFromCA(
         outputPathName(drvName, outputName),
@@ -214,7 +214,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 
 static DerivationOutput parseDerivationOutput(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -261,7 +261,7 @@ static DerivationOutput parseDerivationOutput(
 }
 
 static DerivationOutput parseDerivationOutput(
-    const Store & store, std::istringstream & str,
+    const StoreDirConfig & store, std::istringstream & str,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
@@ -290,7 +290,7 @@ enum struct DerivationATermVersion {
 };
 
 static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
-    const Store & store,
+    const StoreDirConfig & store,
     std::istringstream & str,
     DerivationATermVersion version)
 {
@@ -337,7 +337,7 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
 
 
 Derivation parseDerivation(
-    const Store & store, std::string && s, std::string_view name,
+    const StoreDirConfig & store, std::string && s, std::string_view name,
     const ExperimentalFeatureSettings & xpSettings)
 {
     Derivation drv;
@@ -470,7 +470,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt
 }
 
 
-static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
+static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
 {
     s += ',';
     if (node.childMap.empty()) {
@@ -511,7 +511,7 @@ static bool hasDynamicDrvDep(const Derivation & drv)
 }
 
 
-std::string Derivation::unparse(const Store & store, bool maskOutputs,
+std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
     DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
 {
     std::string s;
@@ -845,7 +845,7 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
 }
 
 
-static DerivationOutput readDerivationOutput(Source & in, const Store & store)
+static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store)
 {
     const auto pathS = readString(in);
     const auto hashAlgo = readString(in);
@@ -862,7 +862,7 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
-DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const
 {
     DerivationOutputsAndOptPaths outsAndOptPaths;
     for (auto & [outputName, output] : outputs)
@@ -884,7 +884,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
 }
 
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
+Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name)
 {
     drv.name = name;
 
@@ -912,7 +912,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
 }
 
 
-void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
+void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs) {
@@ -1153,7 +1153,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
-    const Store & store, std::string_view drvName, OutputNameView outputName) const
+    const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     nlohmann::json res = nlohmann::json::object();
     std::visit(overloaded {
@@ -1180,7 +1180,7 @@ nlohmann::json DerivationOutput::toJSON(
 
 
 DerivationOutput DerivationOutput::fromJSON(
-    const Store & store, std::string_view drvName, OutputNameView outputName,
+    const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName,
     const nlohmann::json & _json,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -1249,7 +1249,7 @@ DerivationOutput DerivationOutput::fromJSON(
 }
 
 
-nlohmann::json Derivation::toJSON(const Store & store) const
+nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res = nlohmann::json::object();
 
@@ -1302,7 +1302,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const
 
 
 Derivation Derivation::fromJSON(
-    const Store & store,
+    const StoreDirConfig & store,
     const nlohmann::json & json,
     const ExperimentalFeatureSettings & xpSettings)
 {

src/libstore/derivations.hh

@@ -17,7 +17,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 
 /* Abstract syntax of derivations. */
 
@@ -55,7 +55,7 @@ struct DerivationOutput
          * @param drvName The name of the derivation this is an output of, without the `.drv`.
          * @param outputName The name of this output.
          */
-        StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
+        StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
 
         GENERATE_CMP(CAFixed, me->ca);
     };
@@ -132,17 +132,17 @@ struct DerivationOutput
      * the safer interface provided by
      * BasicDerivation::outputsAndOptPaths
      */
-    std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
+    std::optional<StorePath> path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
 
     nlohmann::json toJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view drvName,
         OutputNameView outputName) const;
     /**
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivationOutput fromJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view drvName,
         OutputNameView outputName,
         const nlohmann::json & json,
@@ -304,7 +304,7 @@ struct BasicDerivation
      * augmented with knowledge of the Store paths they would be written
      * into.
      */
-    DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
+    DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const;
 
     static std::string_view nameFromPath(const StorePath & storePath);
 
@@ -318,6 +318,8 @@ struct BasicDerivation
         me->name);
 };
 
+class Store;
+
 struct Derivation : BasicDerivation
 {
     /**
@@ -328,7 +330,7 @@ struct Derivation : BasicDerivation
     /**
      * Print a derivation.
      */
-    std::string unparse(const Store & store, bool maskOutputs,
+    std::string unparse(const StoreDirConfig & store, bool maskOutputs,
         DerivedPathMap<StringSet>::ChildNode::Map * actualInputs = nullptr) const;
 
     /**
@@ -365,9 +367,9 @@ struct Derivation : BasicDerivation
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
     static Derivation fromJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         const nlohmann::json & json,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
@@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store,
  * Read a derivation from a file.
  */
 Derivation parseDerivation(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string && s,
     std::string_view name,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
@@ -493,8 +495,8 @@ extern Sync<DrvHashes> drvHashes;
 struct Source;
 struct Sink;
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
+Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name);
-void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
+void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv);
 
 /**
  * This creates an opaque and almost certainly unique string

src/libstore/derived-path.cc

@@ -1,4 +1,5 @@
 #include "derived-path.hh"
+#include "derivations.hh"
 #include "store-api.hh"
 
 #include <nlohmann/json.hpp>
@@ -32,7 +33,7 @@ CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
 #undef CMP
 #undef CMP_ONE
 
-nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const
+nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const
 {
     return store.printStorePath(path);
 }
@@ -86,50 +87,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const
     }, raw());
 }
 
-std::string DerivedPath::Opaque::to_string(const Store & store) const
+std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const
 {
     return store.printStorePath(path);
 }
 
-std::string SingleDerivedPath::Built::to_string(const Store & store) const
+std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store) + "^" + output;
 }
 
-std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const
+std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store) + "!" + output;
 }
 
-std::string DerivedPath::Built::to_string(const Store & store) const
+std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store)
         + '^'
         + outputs.to_string();
 }
 
-std::string DerivedPath::Built::to_string_legacy(const Store & store) const
+std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
 {
     return drvPath->to_string_legacy(store)
         + "!"
         + outputs.to_string();
 }
 
-std::string SingleDerivedPath::to_string(const Store & store) const
+std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
         raw());
 }
 
-std::string DerivedPath::to_string(const Store & store) const
+std::string DerivedPath::to_string(const StoreDirConfig & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
         raw());
 }
 
-std::string SingleDerivedPath::to_string_legacy(const Store & store) const
+std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const
 {
     return std::visit(overloaded {
         [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); },
@@ -137,7 +138,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const
     }, this->raw());
 }
 
-std::string DerivedPath::to_string_legacy(const Store & store) const
+std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
@@ -146,7 +147,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const
 }
 
 
-DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s)
+DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s)
 {
     return {store.parseStorePath(s)};
 }
@@ -166,7 +167,7 @@ void drvRequireExperiment(
 }
 
 SingleDerivedPath::Built SingleDerivedPath::Built::parse(
-    const Store & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
     OutputNameView output,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -178,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
 }
 
 DerivedPath::Built DerivedPath::Built::parse(
-    const Store & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
     OutputNameView outputsS,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -190,7 +191,7 @@ DerivedPath::Built DerivedPath::Built::parse(
 }
 
 static SingleDerivedPath parseWithSingle(
-    const Store & store, std::string_view s, std::string_view separator,
+    const StoreDirConfig & store, std::string_view s, std::string_view separator,
     const ExperimentalFeatureSettings & xpSettings)
 {
     size_t n = s.rfind(separator);
@@ -207,7 +208,7 @@ static SingleDerivedPath parseWithSingle(
 }
 
 SingleDerivedPath SingleDerivedPath::parse(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -215,7 +216,7 @@ SingleDerivedPath SingleDerivedPath::parse(
 }
 
 SingleDerivedPath SingleDerivedPath::parseLegacy(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -223,7 +224,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy(
 }
 
 static DerivedPath parseWith(
-    const Store & store, std::string_view s, std::string_view separator,
+    const StoreDirConfig & store, std::string_view s, std::string_view separator,
     const ExperimentalFeatureSettings & xpSettings)
 {
     size_t n = s.rfind(separator);
@@ -240,7 +241,7 @@ static DerivedPath parseWith(
 }
 
 DerivedPath DerivedPath::parse(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -248,7 +249,7 @@ DerivedPath DerivedPath::parse(
 }
 
 DerivedPath DerivedPath::parseLegacy(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {

src/libstore/derived-path.hh

@@ -12,6 +12,9 @@
 
 namespace nix {
 
+struct StoreDirConfig;
+
+// TODO stop needing this, `toJSON` below should be pure
 class Store;
 
 /**
@@ -24,9 +27,9 @@ class Store;
 struct DerivedPathOpaque {
     StorePath path;
 
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
-    static DerivedPathOpaque parse(const Store & store, std::string_view);
+    static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     GENERATE_CMP(DerivedPathOpaque, me->path);
 };
@@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * The caller splits on the separator, so it works for both variants.
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPathBuilt parse(
-        const Store & store, ref<SingleDerivedPath> drvPath,
+        const StoreDirConfig & store, ref<SingleDerivedPath> drvPath,
         OutputNameView outputs,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * Uses `^` as the separator
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPath parse(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     /**
@@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPath parseLegacy(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -182,18 +185,18 @@ struct DerivedPathBuilt {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * The caller splits on the separator, so it works for both variants.
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPathBuilt parse(
-        const Store & store, ref<SingleDerivedPath>,
+        const StoreDirConfig & store, ref<SingleDerivedPath>,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * Uses `^` as the separator
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPath parse(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     /**
@@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw {
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPath parseLegacy(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 

src/libstore/globals.cc

@@ -116,7 +116,14 @@ Settings::Settings()
 
 void loadConfFile()
 {
-    globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
+    auto applyConfigFile = [&](const Path & path) {
+        try {
+            std::string contents = readFile(path);
+            globalConfig.applyConfig(contents, path);
+        } catch (SysError &) { }
+    };
+
+    applyConfigFile(settings.nixConfDir + "/nix.conf");
 
     /* We only want to send overrides to the daemon, i.e. stuff from
        ~/.nix/nix.conf or the command line. */
@@ -124,7 +131,7 @@ void loadConfFile()
 
     auto files = settings.nixUserConfFiles;
     for (auto file = files.rbegin(); file != files.rend(); file++) {
-        globalConfig.applyConfigFile(*file);
+        applyConfigFile(*file);
     }
 
     auto nixConfEnv = getEnv("NIX_CONFIG");

src/libstore/globals.hh

@@ -117,10 +117,11 @@ public:
 
     Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
         R"(
-          The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
           to use for most operations.
-          See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
+          See the
-          for supported store types and settings.
+          [Store Types](@docroot@/store/types/index.md)
+          section of the manual for supported store types and settings.
         )"};
 
     Setting<bool> keepFailed{this, false, "keep-failed",
@@ -200,7 +201,7 @@ public:
           Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms).
 
           The default value is set when Nix itself is compiled for the system it will run on.
-          The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms):
+          The following system types are widely used, as Nix is actively supported on these platforms:
 
           - `x86_64-linux`
           - `x86_64-darwin`
@@ -759,8 +760,8 @@ public:
         Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
-          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace.
+          A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
-          A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
+          A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
 
           Substituters are tried based on their priority value, which each substituter can set independently.
           Lower value means higher priority.
@@ -778,7 +779,7 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
+          A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace.
           These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters).
 
           Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`.

src/libstore/length-prefixed-protocol-helper.hh

@@ -10,7 +10,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 
 /**
  * Reusable serialisers for serialization container types in a
@@ -44,8 +44,8 @@ struct LengthPrefixedProtoHelper;
 #define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \
     struct LengthPrefixedProtoHelper< Inner, T > \
     { \
-        static T read(const Store & store, typename Inner::ReadConn conn); \
+        static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \
-        static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \
+        static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \
     private: \
         template<typename U> using S = typename Inner::template Serialise<U>; \
     }
@@ -67,7 +67,7 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, _X);
 template<class Inner, typename T>
 std::vector<T>
 LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::vector<T> resSet;
     auto size = readNum<size_t>(conn.from);
@@ -80,7 +80,7 @@ LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
 template<class Inner, typename T>
 void
 LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
 {
     conn.to << resSet.size();
     for (auto & key : resSet) {
@@ -91,7 +91,7 @@ LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
 template<class Inner, typename T>
 std::set<T>
 LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::set<T> resSet;
     auto size = readNum<size_t>(conn.from);
@@ -104,7 +104,7 @@ LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
 template<class Inner, typename T>
 void
 LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
 {
     conn.to << resSet.size();
     for (auto & key : resSet) {
@@ -115,7 +115,7 @@ LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
 template<class Inner, typename K, typename V>
 std::map<K, V>
 LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::map<K, V> resMap;
     auto size = readNum<size_t>(conn.from);
@@ -130,7 +130,7 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
 template<class Inner, typename K, typename V>
 void
 LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
 {
     conn.to << resMap.size();
     for (auto & i : resMap) {
@@ -142,7 +142,7 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
 template<class Inner, typename... Ts>
 std::tuple<Ts...>
 LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     return std::tuple<Ts...> {
         S<Ts>::read(store, conn)...,
@@ -152,7 +152,7 @@ LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
 template<class Inner, typename... Ts>
 void
 LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
 {
     std::apply([&]<typename... Us>(const Us &... args) {
         (S<Us>::write(store, conn, args), ...);

src/libstore/local-store.cc

@@ -4,6 +4,7 @@
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
 #include "derivations.hh"
+#include "realisation.hh"
 #include "nar-info.hh"
 #include "references.hh"
 #include "callback.hh"
@@ -34,7 +35,6 @@
 #include <sys/statvfs.h>
 #include <sys/mount.h>
 #include <sys/ioctl.h>
-#include <sys/xattr.h>
 #endif
 
 #ifdef __CYGWIN__

src/libstore/local.mk

@@ -59,7 +59,7 @@ $(d)/build.cc:
 
 clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
-$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libstore/builtins/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))

src/libstore/misc.cc

@@ -4,6 +4,7 @@
 #include "local-store.hh"
 #include "store-api.hh"
 #include "thread-pool.hh"
+#include "realisation.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "closure.hh"

src/libstore/path-with-outputs.cc

@@ -5,7 +5,7 @@
 
 namespace nix {
 
-std::string StorePathWithOutputs::to_string(const Store & store) const
+std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const
 {
     return outputs.empty()
         ? store.printStorePath(path)
@@ -85,7 +85,7 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
 }
 
 
-StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
+StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs)
 {
     auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
     return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) };

src/libstore/path-with-outputs.hh

@@ -6,6 +6,8 @@
 
 namespace nix {
 
+struct StoreDirConfig;
+
 /**
  * This is a deprecated old type just for use by the old CLI, and older
  * versions of the RPC protocols. In new code don't use it; you want
@@ -19,7 +21,7 @@ struct StorePathWithOutputs
     StorePath path;
     std::set<std::string> outputs;
 
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
 
     DerivedPath toDerivedPath() const;
 
@@ -32,14 +34,14 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>)
 
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
 
-class Store;
-
 /**
  * Split a string specifying a derivation and a set of outputs
  * (/nix/store/hash-foo!out1,out2,...) into the derivation path
  * and the outputs.
  */
-StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
+StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs);
+
+class Store;
 
 StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
 

src/libstore/path.cc

@@ -1,4 +1,4 @@
-#include "store-api.hh"
+#include "store-dir-config.hh"
 
 #include <sodium.h>
 
@@ -54,7 +54,7 @@ StorePath StorePath::random(std::string_view name)
     return StorePath(hash, name);
 }
 
-StorePath Store::parseStorePath(std::string_view path) const
+StorePath StoreDirConfig::parseStorePath(std::string_view path) const
 {
     auto p = canonPath(std::string(path));
     if (dirOf(p) != storeDir)
@@ -62,7 +62,7 @@ StorePath Store::parseStorePath(std::string_view path) const
     return StorePath(baseNameOf(p));
 }
 
-std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
+std::optional<StorePath> StoreDirConfig::maybeParseStorePath(std::string_view path) const
 {
     try {
         return parseStorePath(path);
@@ -71,24 +71,24 @@ std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
     }
 }
 
-bool Store::isStorePath(std::string_view path) const
+bool StoreDirConfig::isStorePath(std::string_view path) const
 {
     return (bool) maybeParseStorePath(path);
 }
 
-StorePathSet Store::parseStorePathSet(const PathSet & paths) const
+StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const
 {
     StorePathSet res;
     for (auto & i : paths) res.insert(parseStorePath(i));
     return res;
 }
 
-std::string Store::printStorePath(const StorePath & path) const
+std::string StoreDirConfig::printStorePath(const StorePath & path) const
 {
     return (storeDir + "/").append(path.to_string());
 }
 
-PathSet Store::printStorePathSet(const StorePathSet & paths) const
+PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const
 {
     PathSet res;
     for (auto & i : paths) res.insert(printStorePath(i));

src/libstore/posix-fs-canonicalise.cc

@@ -1,4 +1,6 @@
-#include <sys/xattr.h>
+#if HAVE_SYS_XATTR_H
+# include <sys/xattr.h>
+#endif
 
 #include "posix-fs-canonicalise.hh"
 #include "file-system.hh"
@@ -76,7 +78,7 @@ static void canonicalisePathMetaData_(
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error("file '%1%' has an unsupported type", path);
 
-#if __linux__
+#ifdef HAVE_SYS_XATTR_H
     /* Remove extended attributes / ACLs. */
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 

src/libstore/remote-store.cc

@@ -20,7 +20,7 @@
 
 namespace nix {
 
-/* TODO: Separate these store impls into different files, give them better names */
+/* TODO: Separate these store types into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : RemoteStoreConfig(params)
     , Store(params)

src/libstore/serve-protocol-impl.hh

@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \
+    TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<ServeProto, T >::read(store, conn); \
     } \
-    TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \
+    TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<ServeProto, T >::write(store, conn, t); \
     }
@@ -41,12 +41,12 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(
 template<typename T>
 struct ServeProto::Serialise
 {
-    static T read(const Store & store, ServeProto::ReadConn conn)
+    static T read(const StoreDirConfig & store, ServeProto::ReadConn conn)
     {
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, ServeProto::WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },

src/libstore/serve-protocol.cc

@@ -12,7 +12,7 @@ namespace nix {
 
 /* protocol-specific definitions */
 
-BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeProto::ReadConn conn)
+BuildResult ServeProto::Serialise<BuildResult>::read(const StoreDirConfig & store, ServeProto::ReadConn conn)
 {
     BuildResult status;
     status.status = (BuildResult::Status) readInt(conn.from);
@@ -34,7 +34,7 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeP
     return status;
 }
 
-void ServeProto::Serialise<BuildResult>::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status)
+void ServeProto::Serialise<BuildResult>::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status)
 {
     conn.to
         << status.status

src/libstore/serve-protocol.hh

@@ -13,7 +13,7 @@ namespace nix {
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialised
@@ -72,8 +72,8 @@ struct ServeProto
     // See `worker-protocol.hh` for a longer explanation.
 #if 0
     {
-        static T read(const Store & store, ReadConn conn);
+        static T read(const StoreDirConfig & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static void write(const StoreDirConfig & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -82,7 +82,7 @@ struct ServeProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         ServeProto::Serialise<T>::write(store, conn, t);
     }
@@ -135,8 +135,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
 #define DECLARE_SERVE_SERIALISER(T) \
     struct ServeProto::Serialise< T > \
     { \
-        static T read(const Store & store, ServeProto::ReadConn conn); \
+        static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \
-        static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \
+        static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \
     };
 
 template<>

src/libstore/ssh-store-config.hh

@@ -20,7 +20,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig
 
     const Setting<std::string> remoteStore{this, "", "remote-store",
         R"(
-          [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          [Store URL](@docroot@/store/types/index.md#store-url-format)
           to be used on the remote machine. The default is `auto`
           (i.e. use the Nix daemon or `/nix/store` directly).
         )"};

src/libstore/store-api.cc

@@ -1,6 +1,8 @@
 #include "crypto.hh"
 #include "source-accessor.hh"
 #include "globals.hh"
+#include "derived-path.hh"
+#include "realisation.hh"
 #include "derivations.hh"
 #include "store-api.hh"
 #include "util.hh"
@@ -25,13 +27,13 @@ using json = nlohmann::json;
 namespace nix {
 
 
-bool Store::isInStore(PathView path) const
+bool StoreDirConfig::isInStore(PathView path) const
 {
     return isInDir(path, storeDir);
 }
 
 
-std::pair<StorePath, Path> Store::toStorePath(PathView path) const
+std::pair<StorePath, Path> StoreDirConfig::toStorePath(PathView path) const
 {
     if (!isInStore(path))
         throw Error("path '%1%' is not in the Nix store", path);
@@ -145,7 +147,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
 */
 
 
-StorePath Store::makeStorePath(std::string_view type,
+StorePath StoreDirConfig::makeStorePath(std::string_view type,
     std::string_view hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
@@ -156,14 +158,14 @@ StorePath Store::makeStorePath(std::string_view type,
 }
 
 
-StorePath Store::makeStorePath(std::string_view type,
+StorePath StoreDirConfig::makeStorePath(std::string_view type,
     const Hash & hash, std::string_view name) const
 {
     return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name);
 }
 
 
-StorePath Store::makeOutputPath(std::string_view id,
+StorePath StoreDirConfig::makeOutputPath(std::string_view id,
     const Hash & hash, std::string_view name) const
 {
     return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
@@ -174,7 +176,7 @@ StorePath Store::makeOutputPath(std::string_view id,
    hacky, but we can't put them in, say, <s2> (per the grammar above)
    since that would be ambiguous. */
 static std::string makeType(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string && type,
     const StoreReferences & references)
 {
@@ -187,7 +189,7 @@ static std::string makeType(
 }
 
 
-StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
+StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
 {
     if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
@@ -203,7 +205,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
 }
 
 
-StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
+StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
 {
     assert(info.hash.type == htSHA256);
     return makeStorePath(
@@ -216,7 +218,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons
 }
 
 
-StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
+StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
 {
     // New template
     return std::visit(overloaded {
@@ -230,7 +232,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
 }
 
 
-std::pair<StorePath, Hash> Store::computeStorePathFromDump(
+std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
     Source & dump,
     std::string_view name,
     FileIngestionMethod method,
@@ -249,7 +251,7 @@ std::pair<StorePath, Hash> Store::computeStorePathFromDump(
 }
 
 
-StorePath Store::computeStorePathForText(
+StorePath StoreDirConfig::computeStorePathForText(
     std::string_view name,
     std::string_view s,
     const StorePathSet & references) const
@@ -1227,7 +1229,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
 }
 
 
-std::string Store::showPaths(const StorePathSet & paths)
+std::string StoreDirConfig::showPaths(const StorePathSet & paths)
 {
     std::string s;
     for (auto & i : paths) {

src/libstore/store-api.hh

@@ -1,8 +1,6 @@
 #pragma once
 ///@file
 
-#include "nar-info.hh"
-#include "realisation.hh"
 #include "path.hh"
 #include "derived-path.hh"
 #include "hash.hh"
@@ -14,6 +12,7 @@
 #include "config.hh"
 #include "path-info.hh"
 #include "repair-flag.hh"
+#include "store-dir-config.hh"
 
 #include <nlohmann/json_fwd.hpp>
 #include <atomic>
@@ -30,7 +29,7 @@
 namespace nix {
 
 /**
- * About the class hierarchy of the store implementations:
+ * About the class hierarchy of the store types:
  *
  * Each store type `Foo` consists of two classes:
  *
@@ -64,12 +63,16 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
-MakeError(BadStorePath, Error);
 
 MakeError(InvalidStoreURI, Error);
 
+struct Realisation;
+struct RealisedPath;
+struct DrvOutput;
+
 struct BasicDerivation;
 struct Derivation;
+
 struct SourceAccessor;
 class NarInfoDiskCache;
 class Store;
@@ -96,11 +99,11 @@ struct KeyedBuildResult;
 
 typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 
-struct StoreConfig : public Config
+struct StoreConfig : public StoreDirConfig
 {
     typedef std::map<std::string, std::string> Params;
 
-    using Config::Config;
+    using StoreDirConfig::StoreDirConfig;
 
     StoreConfig() = delete;
 
@@ -130,15 +133,6 @@ struct StoreConfig : public Config
         return std::nullopt;
     }
 
-    const PathSetting storeDir_{this, settings.nixStore,
-        "store",
-        R"(
-          Logical location of the Nix store, usually
-          `/nix/store`. Note that you can only copy store paths
-          between stores if they have the same `store` setting.
-        )"};
-    const Path storeDir = storeDir_;
-
     const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
         "Size of the in-memory store path metadata cache."};
 
@@ -226,45 +220,6 @@ public:
 
     virtual std::string getUri() = 0;
 
-    StorePath parseStorePath(std::string_view path) const;
-
-    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
-
-    std::string printStorePath(const StorePath & path) const;
-
-    /**
-     * Deprecated
-     *
-     * \todo remove
-     */
-    StorePathSet parseStorePathSet(const PathSet & paths) const;
-
-    PathSet printStorePathSet(const StorePathSet & path) const;
-
-    /**
-     * Display a set of paths in human-readable form (i.e., between quotes
-     * and separated by commas).
-     */
-    std::string showPaths(const StorePathSet & paths);
-
-    /**
-     * @return true if ‘path’ is in the Nix store (but not the Nix
-     * store itself).
-     */
-    bool isInStore(PathView path) const;
-
-    /**
-     * @return true if ‘path’ is a store path, i.e. a direct child of the
-     * Nix store.
-     */
-    bool isStorePath(std::string_view path) const;
-
-    /**
-     * Split a path like /nix/store/<hash>-<name>/<bla> into
-     * /nix/store/<hash>-<name> and /<bla>.
-     */
-    std::pair<StorePath, Path> toStorePath(PathView path) const;
-
     /**
      * Follow symlinks until we end up with a path in the Nix store.
      */
@@ -276,55 +231,6 @@ public:
      */
     StorePath followLinksToStorePath(std::string_view path) const;
 
-    /**
-     * Constructs a unique store path name.
-     */
-    StorePath makeStorePath(std::string_view type,
-        std::string_view hash, std::string_view name) const;
-    StorePath makeStorePath(std::string_view type,
-        const Hash & hash, std::string_view name) const;
-
-    StorePath makeOutputPath(std::string_view id,
-        const Hash & hash, std::string_view name) const;
-
-    StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
-
-    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
-
-    StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
-
-    /**
-     * Read-only variant of addToStoreFromDump(). It returns the store
-     * path to which a NAR or flat file would be written.
-     */
-    std::pair<StorePath, Hash> computeStorePathFromDump(
-        Source & dump,
-        std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        HashType hashAlgo = htSHA256,
-        const StorePathSet & references = {}) const;
-
-    /**
-     * Preparatory part of addTextToStore().
-     *
-     * !!! Computation of the path should take the references given to
-     * addTextToStore() into account, otherwise we have a (relatively
-     * minor) security hole: a caller can register a source file with
-     * bogus references.  If there are too many references, the path may
-     * not be garbage collected when it has to be (not really a problem,
-     * the caller could create a root anyway), or it may be garbage
-     * collected when it shouldn't be (more serious).
-     *
-     * Hashing the references would solve this (bogus references would
-     * simply yield a different store path, so other users wouldn't be
-     * affected), but it has some backwards compatibility issues (the
-     * hashing scheme changes), so I'm not doing that for now.
-     */
-    StorePath computeStorePathForText(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references) const;
-
     /**
      * Check whether a path is valid.
      */
@@ -888,7 +794,7 @@ void copyStorePath(
  */
 std::map<StorePath, StorePath> copyPaths(
     Store & srcStore, Store & dstStore,
-    const RealisedPath::Set &,
+    const std::set<RealisedPath> &,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
@@ -905,7 +811,7 @@ std::map<StorePath, StorePath> copyPaths(
  */
 void copyClosure(
     Store & srcStore, Store & dstStore,
-    const RealisedPath::Set & paths,
+    const std::set<RealisedPath> & paths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
@@ -962,7 +868,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
  * - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
  *   ‘nix-store --serve’ via SSH.
  *
- * You can pass parameters to the store implementation by appending
+ * You can pass parameters to the store type by appending
  * ‘?key=value&key=value&...’ to the URI.
  */
 ref<Store> openStore(const std::string & uri = settings.storeUri.get(),

src/libstore/store-dir-config.hh

@@ -0,0 +1,126 @@
+#pragma once
+
+#include "path.hh"
+#include "hash.hh"
+#include "content-address.hh"
+#include "globals.hh"
+#include "config.hh"
+
+#include <map>
+#include <string>
+#include <variant>
+
+
+namespace nix {
+
+MakeError(BadStorePath, Error);
+
+struct StoreDirConfig : public Config
+{
+    using Config::Config;
+
+    StoreDirConfig() = delete;
+
+    virtual ~StoreDirConfig() = default;
+
+    const PathSetting storeDir_{this, settings.nixStore,
+        "store",
+        R"(
+          Logical location of the Nix store, usually
+          `/nix/store`. Note that you can only copy store paths
+          between stores if they have the same `store` setting.
+        )"};
+    const Path storeDir = storeDir_;
+
+    // pure methods
+
+    StorePath parseStorePath(std::string_view path) const;
+
+    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
+
+    std::string printStorePath(const StorePath & path) const;
+
+    /**
+     * Deprecated
+     *
+     * \todo remove
+     */
+    StorePathSet parseStorePathSet(const PathSet & paths) const;
+
+    PathSet printStorePathSet(const StorePathSet & path) const;
+
+    /**
+     * Display a set of paths in human-readable form (i.e., between quotes
+     * and separated by commas).
+     */
+    std::string showPaths(const StorePathSet & paths);
+
+    /**
+     * @return true if ‘path’ is in the Nix store (but not the Nix
+     * store itself).
+     */
+    bool isInStore(PathView path) const;
+
+    /**
+     * @return true if ‘path’ is a store path, i.e. a direct child of the
+     * Nix store.
+     */
+    bool isStorePath(std::string_view path) const;
+
+    /**
+     * Split a path like /nix/store/<hash>-<name>/<bla> into
+     * /nix/store/<hash>-<name> and /<bla>.
+     */
+    std::pair<StorePath, Path> toStorePath(PathView path) const;
+
+    /**
+     * Constructs a unique store path name.
+     */
+    StorePath makeStorePath(std::string_view type,
+        std::string_view hash, std::string_view name) const;
+    StorePath makeStorePath(std::string_view type,
+        const Hash & hash, std::string_view name) const;
+
+    StorePath makeOutputPath(std::string_view id,
+        const Hash & hash, std::string_view name) const;
+
+    StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
+
+    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
+
+    StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
+
+    /**
+     * Read-only variant of addToStoreFromDump(). It returns the store
+     * path to which a NAR or flat file would be written.
+     */
+    std::pair<StorePath, Hash> computeStorePathFromDump(
+        Source & dump,
+        std::string_view name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        HashType hashAlgo = htSHA256,
+        const StorePathSet & references = {}) const;
+
+    /**
+     * Preparatory part of addTextToStore().
+     *
+     * !!! Computation of the path should take the references given to
+     * addTextToStore() into account, otherwise we have a (relatively
+     * minor) security hole: a caller can register a source file with
+     * bogus references.  If there are too many references, the path may
+     * not be garbage collected when it has to be (not really a problem,
+     * the caller could create a root anyway), or it may be garbage
+     * collected when it shouldn't be (more serious).
+     *
+     * Hashing the references would solve this (bogus references would
+     * simply yield a different store path, so other users wouldn't be
+     * affected), but it has some backwards compatibility issues (the
+     * hashing scheme changes), so I'm not doing that for now.
+     */
+    StorePath computeStorePathForText(
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references) const;
+};
+
+}

src/libstore/tests/local.mk

@@ -1,37 +0,0 @@
-check: libstore-tests-exe_RUN
-
-programs += libstore-tests-exe
-
-libstore-tests-exe_NAME = libnixstore-tests
-
-libstore-tests-exe_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-tests-exe_INSTALL_DIR := $(checkbindir)
-else
-  libstore-tests-exe_INSTALL_DIR :=
-endif
-
-libstore-tests-exe_LIBS = libstore-tests
-
-libstore-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libstore-tests
-
-libstore-tests_NAME = libnixstore-tests
-
-libstore-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-tests_INSTALL_DIR := $(checklibdir)
-else
-  libstore-tests_INSTALL_DIR :=
-endif
-
-libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
-
-libstore-tests_LIBS = libutil-tests libstore libutil
-
-libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

src/libstore/worker-protocol-impl.hh

@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \
+    TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<WorkerProto, T >::read(store, conn); \
     } \
-    TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \
+    TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<WorkerProto, T >::write(store, conn, t); \
     }
@@ -41,12 +41,12 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(
 template<typename T>
 struct WorkerProto::Serialise
 {
-    static T read(const Store & store, WorkerProto::ReadConn conn)
+    static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
     {
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, WorkerProto::WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },

src/libstore/worker-protocol.cc

@@ -13,7 +13,7 @@ namespace nix {
 
 /* protocol-specific definitions */
 
-std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, WorkerProto::ReadConn conn)
+std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto temp = readNum<uint8_t>(conn.from);
     switch (temp) {
@@ -28,7 +28,7 @@ std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::r
     }
 }
 
-void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
+void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
         conn.to << uint8_t{0};
@@ -47,7 +47,7 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & sto
 }
 
 
-DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, WorkerProto::ReadConn conn)
+DerivedPath WorkerProto::Serialise<DerivedPath>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto s = readString(conn.from);
     if (GET_PROTOCOL_MINOR(conn.version) >= 30) {
@@ -57,7 +57,7 @@ DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, Worke
     }
 }
 
-void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req)
+void WorkerProto::Serialise<DerivedPath>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req)
 {
     if (GET_PROTOCOL_MINOR(conn.version) >= 30) {
         conn.to << req.to_string_legacy(store);
@@ -81,7 +81,7 @@ void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto
 }
 
 
-KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
+KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto path = WorkerProto::Serialise<DerivedPath>::read(store, conn);
     auto br = WorkerProto::Serialise<BuildResult>::read(store, conn);
@@ -91,14 +91,14 @@ KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & st
     };
 }
 
-void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
+void WorkerProto::Serialise<KeyedBuildResult>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
 {
     WorkerProto::write(store, conn, res.path);
     WorkerProto::write(store, conn, static_cast<const BuildResult &>(res));
 }
 
 
-BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
+BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     BuildResult res;
     res.status = static_cast<BuildResult::Status>(readInt(conn.from));
@@ -120,7 +120,7 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, Worke
     return res;
 }
 
-void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res)
+void WorkerProto::Serialise<BuildResult>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res)
 {
     conn.to
         << res.status
@@ -141,7 +141,7 @@ void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto
 }
 
 
-ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const Store & store, ReadConn conn)
+ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto path = WorkerProto::Serialise<StorePath>::read(store, conn);
     return ValidPathInfo {
@@ -150,14 +150,14 @@ ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const Store & store, R
     };
 }
 
-void WorkerProto::Serialise<ValidPathInfo>::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo)
+void WorkerProto::Serialise<ValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo)
 {
     WorkerProto::write(store, conn, pathInfo.path);
     WorkerProto::write(store, conn, static_cast<const UnkeyedValidPathInfo &>(pathInfo));
 }
 
 
-UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const Store & store, ReadConn conn)
+UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto deriver = readString(conn.from);
     auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
@@ -173,7 +173,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const St
     return info;
 }
 
-void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
+void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
 {
     conn.to
         << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "")

src/libstore/worker-protocol.hh

@@ -24,7 +24,7 @@ namespace nix {
 #define STDERR_RESULT         0x52534c54
 
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialised
@@ -100,8 +100,8 @@ struct WorkerProto
     // This makes for a quicker debug cycle, as desired.
 #if 0
     {
-        static T read(const Store & store, ReadConn conn);
+        static T read(const StoreDirConfig & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static void write(const StoreDirConfig & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -110,7 +110,7 @@ struct WorkerProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         WorkerProto::Serialise<T>::write(store, conn, t);
     }
@@ -198,8 +198,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
 #define DECLARE_WORKER_SERIALISER(T) \
     struct WorkerProto::Serialise< T > \
     { \
-        static T read(const Store & store, WorkerProto::ReadConn conn); \
+        static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \
-        static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
+        static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \
     };
 
 template<>

src/libutil/config.cc

@@ -88,10 +88,9 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridd
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
-void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
-    unsigned int pos = 0;
 
-    std::vector<std::pair<std::string, std::string>> parsedContents;
+static void applyConfigInner(const std::string & contents, const std::string & path, std::vector<std::pair<std::string, std::string>> & parsedContents) {
+    unsigned int pos = 0;
 
     while (pos < contents.size()) {
         std::string line;
@@ -122,7 +121,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
                 throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
             auto p = absPath(tokens[1], dirOf(path));
             if (pathExists(p)) {
-                applyConfigFile(p);
+                try {
+                    std::string includedContents = readFile(path);
+                    applyConfigInner(includedContents, p, parsedContents);
+                } catch (SysError &) {
+                    // TODO: Do we actually want to ignore this? Or is it better to fail?
+                }
             } else if (!ignoreMissing) {
                 throw Error("file '%1%' included from '%2%' not found", p, path);
             }
@@ -142,6 +146,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             concatStringsSep(" ", Strings(i, tokens.end())),
         });
     };
+}
+
+void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
+    std::vector<std::pair<std::string, std::string>> parsedContents;
+
+    applyConfigInner(contents, path, parsedContents);
 
     // First apply experimental-feature related settings
     for (const auto & [name, value] : parsedContents)
@@ -154,14 +164,6 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             set(name, value);
 }
 
-void AbstractConfig::applyConfigFile(const Path & path)
-{
-    try {
-        std::string contents = readFile(path);
-        applyConfig(contents, path);
-    } catch (SysError &) { }
-}
-
 void Config::resetOverridden()
 {
     for (auto & s : _settings)