Merge remote-tracking branch 'upstream/master' into upstream-merge

.gitignore

@@ -90,6 +90,7 @@ perl/Makefile.config
 
 /misc/systemd/nix-daemon.service
 /misc/systemd/nix-daemon.socket
+/misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 
 /src/resolve-system-dependencies/resolve-system-dependencies

.version

@@ -1 +1 @@
-2.7.0
+2.8.0

doc/manual/generate-builtins.nix

@@ -6,9 +6,9 @@ builtins:
 concatStrings (map
   (name:
     let builtin = builtins.${name}; in
-    "<dt><code>${name} "
+    "<dt id=\"builtins-${name}\"><a href=\"#builtins-${name}\"><code>${name} "
     + concatStringsSep " " (map (s: "<var>${s}</var>") builtin.args)
-    + "</code></dt>"
+    + "</code></a></dt>"
     + "<dd>\n\n"
     + builtin.doc
     + "\n\n</dd>"

doc/manual/generate-options.nix

@@ -6,7 +6,8 @@ options:
 concatStrings (map
   (name:
     let option = options.${name}; in
-    "  - `${name}`  \n\n"
+    "  - [`${name}`](#conf-${name})"
+    + "<p id=\"conf-${name}\"></p>\n\n"
     + concatStrings (map (s: "    ${s}\n") (splitLines option.description)) + "\n\n"
     + (if option.documentDefault
        then "    **Default:** " + (
@@ -20,7 +21,7 @@ concatStrings (map
          # JSON, but that converts to "{ }" here.
          (if isAttrs option.value then "`\"\"`"
           else "`" + toString option.value + "`")) + "\n\n"
-       else "    **Default:** *machine-specific*")
+       else "    **Default:** *machine-specific*\n")
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
        else "")

doc/manual/local.mk

@@ -72,6 +72,7 @@ $(d)/builtins.json: $(bindir)/nix
 	@mv $@.tmp $@
 
 # Generate the HTML manual.
+html: $(docdir)/manual/index.html
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.

doc/manual/src/SUMMARY.md.in

@@ -72,6 +72,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md)
   - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md)
   - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md)
   - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)

doc/manual/src/advanced-topics/distributed-builds.md

@@ -110,7 +110,7 @@ default, set it to `-`.
 7.  A comma-separated list of *mandatory features*. A machine will only
     be used to build a derivation if all of the machine’s mandatory
     features appear in the derivation’s `requiredSystemFeatures`
-    attribute..
+    attribute.
 
 8.  The (base64-encoded) public host key of the remote machine. If omitted, SSH
     will use its regular known-hosts file. Specifically, the field is calculated

doc/manual/src/command-ref/nix-store.md

@@ -321,8 +321,8 @@ symlink.
     This query has one option:
 
       - `--include-outputs`
-        Also include the output path of store derivations, and their
+        Also include the existing output paths of store derivations,
-        closures.
+        and their closures.
 
     This query can be used to implement various kinds of deployment. A
     *source deployment* is obtained by distributing the closure of a

doc/manual/src/installation/installing-binary.md

@@ -84,7 +84,9 @@ The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
 The installer will first back up these files with a `.backup-before-nix`
 extension. The installer will also create `/etc/profile.d/nix.sh`.
 
-You can uninstall Nix with the following commands:
+## Uninstalling
+
+### Linux
 
 ```console
 sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
@@ -95,15 +97,95 @@ sudo systemctl stop nix-daemon.service
 sudo systemctl disable nix-daemon.socket
 sudo systemctl disable nix-daemon.service
 sudo systemctl daemon-reload
-
-# If you are on macOS, you will need to run:
-sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 ```
 
 There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
 and `/etc/zshrc` which you may remove.
 
+### macOS
+
+1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing
+   `nix-daemon.sh`, which should look like this:
+
+   ```bash
+   # Nix
+   if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then
+     . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh'
+   fi
+   # End Nix
+   ```
+
+   If these files haven't been altered since installing Nix you can simply put
+   the backups back in place:
+
+   ```console
+   sudo mv /etc/zshrc.backup-before-nix /etc/zshrc
+   sudo mv /etc/bashrc.backup-before-nix /etc/bashrc
+   ```
+
+   This will stop shells from sourcing the file and bringing everything you
+   installed using Nix in scope.
+
+2. Stop and remove the Nix daemon services:
+
+   ```console
+   sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+   sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+   sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist
+   sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist
+   ```
+
+   This stops the Nix daemon and prevents it from being started next time you
+   boot the system.
+
+3. Remove the `nixbld` group and the `_nixbuildN` users:
+
+   ```console
+   sudo dscl . -delete /Groups/nixbld
+   for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done
+   ```
+
+   This will remove all the build users that no longer serve a purpose.
+
+4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store
+   volume on `/nix`, which looks like this,
+   `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic
+   mounting of the Nix Store volume.
+
+5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only
+   line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`.
+   This will prevent the creation of the empty `/nix` directory to provide a
+   mountpoint for the Nix Store volume.
+
+6. Remove the files Nix added to your system:
+
+   ```console
+   sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+   ```
+
+   This gets rid of any data Nix may have created except for the store which is
+   removed next.
+
+7. Remove the Nix Store volume:
+   
+   ```console
+   sudo diskutil apfs deleteVolume /nix
+   ```
+
+   This will remove the Nix Store volume and everything that was added to the
+   store.
+
+> **Note**
+> 
+> After you complete the steps here, you will still have an empty `/nix`
+> directory. This is an expected sign of a successful uninstall. The empty
+> `/nix` directory will disappear the next time you reboot.
+>
+> You do not have to reboot to finish uninstalling Nix. The uninstall is
+> complete. macOS (Catalina+) directly controls root directories and its
+> read-only root will prevent you from manually deleting the empty `/nix`
+> mountpoint.
+
 # macOS Installation <a name="sect-macos-installation-change-store-prefix"></a><a name="sect-macos-installation-encrypted-volume"></a><a name="sect-macos-installation-symlink"></a><a name="sect-macos-installation-recommended-notes"></a>
 <!-- Note: anchors above to catch permalinks to old explanations -->
 

doc/manual/src/release-notes/rl-2.7.md

@@ -0,0 +1,33 @@
+# Release 2.7 (2022-03-07)
+
+* Nix will now make some helpful suggestions when you mistype
+  something on the command line. For instance, if you type `nix build
+  nixpkgs#thunderbrd`, it will suggest `thunderbird`.
+
+* A number of "default" flake output attributes have been
+  renamed. These are:
+
+  * `defaultPackage.<system>` → `packages.<system>.default`
+  * `defaultApps.<system>` → `apps.<system>.default`
+  * `defaultTemplate` → `templates.default`
+  * `defaultBundler.<system>` → `bundlers.<system>.default`
+  * `overlay` → `overlays.default`
+  * `devShell.<system>` → `devShells.<system>.default`
+
+  The old flake output attributes still work, but `nix flake check`
+  will warn about them.
+
+* Breaking API change: `nix bundle` now supports bundlers of the form
+  `bundler.<system>.<name>= derivation: another-derivation;`. This
+  supports additional functionality to inspect evaluation information
+  during bundling. A new
+  [repository](https://github.com/NixOS/bundlers) has various bundlers
+  implemented.
+
+* `nix store ping` now reports the version of the remote Nix daemon.
+
+* `nix flake {init,new}` now display information about which files have been
+  created.
+
+* Templates can now define a `welcomeText` attribute, which is printed out by
+  `nix flake {init,new} --template <template>`.

doc/manual/src/release-notes/rl-next.md

@@ -1,9 +1,42 @@
 # Release X.Y (202?-??-??)
 
-* `nix bundle` breaking API change now supports bundlers of the form
+* Various nix commands can now read expressions from stdin with `--file -`.
-  `bundler.<system>.<name>= derivation: another-derivation;`. This supports
-  additional functionality to inspect evaluation information during bundling. A
-  new [repository](https://github.com/NixOS/bundlers) has various bundlers
-  implemented.
 
-* `nix store ping` now reports the version of the remote Nix daemon.
+* `nix store make-content-addressable` has been renamed to `nix store
+  make-content-addressed`.
+
+* New experimental builtin function `builtins.fetchClosure` that
+  copies a closure from a binary cache at evaluation time and rewrites
+  it to content-addressed form (if it isn't already). Like
+  `builtins.storePath`, this allows importing pre-built store paths;
+  the difference is that it doesn't require the user to configure
+  binary caches and trusted public keys.
+
+  This function is only available if you enable the experimental
+  feature `fetch-closure`.
+
+* New experimental feature: *impure derivations*. These are
+  derivations that can produce a different result every time they're
+  built. Here is an example:
+
+  ```nix
+  stdenv.mkDerivation {
+    name = "impure";
+    __impure = true; # marks this derivation as impure
+    buildCommand = "date > $out";
+  }
+  ```
+
+  Running `nix build` twice on this expression will build the
+  derivation twice, producing two different content-addressed store
+  paths. Like fixed-output derivations, impure derivations have access
+  to the network. Only fixed-output derivations and impure derivations
+  can depend on an impure derivation.
+
+* The `nixosModule` flake output attribute has been renamed consistent
+  with the `.default` renames in nix 2.7.
+
+  * `nixosModule` → `nixosModules.default`
+
+  As before, the old output will continue to work, but `nix flake check` will
+  issue a warning about it.

flake.lock

@@ -18,11 +18,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1632864508,
+        "lastModified": 1645296114,
-        "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
+        "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
+        "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
         "type": "github"
       },
       "original": {

flake.nix

@@ -501,6 +501,12 @@
           inherit (self) overlay;
         });
 
+        tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
         tests.setuid = nixpkgs.lib.genAttrs
           ["i686-linux" "x86_64-linux"]
           (system:
@@ -651,8 +657,7 @@
         nixpkgsFor.${system}.lib.nameValuePair
           "nix-${stdenvName}"
           nixpkgsFor.${system}."${stdenvName}Packages".nix
-      ) stdenvs))
+      ) stdenvs)));
-      );
 
       defaultPackage = forAllSystems (system: self.packages.${system}.nix);
 

maintainers/upload-release.pl

@@ -55,6 +55,11 @@ my $releaseDir = "nix/$releaseName";
 my $tmpDir = "$TMPDIR/nix-release/$releaseName";
 File::Path::make_path($tmpDir);
 
+my $narCache = "$TMPDIR/nar-cache";
+File::Path::make_path($narCache);
+
+my $binaryCache = "https://cache.nixos.org/?local-nar-cache=$narCache";
+
 # S3 setup.
 my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
 my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
@@ -80,6 +85,7 @@ sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
 
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+    #print STDERR "$jobName: ", Dumper($buildInfo), "\n";
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
@@ -87,19 +93,27 @@ sub downloadFile {
 
     if (!-e $tmpFile) {
         print STDERR "downloading $srcFile to $tmpFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix store cat '$srcFile' > '$tmpFile'") == 0
+
+        my $fileInfo = decode_json(`NIX_REMOTE=$binaryCache nix store ls --json '$srcFile'`);
+
+        $srcFile = $fileInfo->{target} if $fileInfo->{type} eq 'symlink';
+
+        #print STDERR $srcFile, " ", Dumper($fileInfo), "\n";
+
+        system("NIX_REMOTE=$binaryCache nix store cat '$srcFile' > '$tmpFile'.tmp") == 0
             or die "unable to fetch $srcFile\n";
+        rename("$tmpFile.tmp", $tmpFile) or die;
     }
 
-    my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
+    my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash};
     my $sha256_actual = `nix hash file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
-    if ($sha256_expected ne $sha256_actual) {
+    if (defined($sha256_expected) && $sha256_expected ne $sha256_actual) {
         print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$tmpFile.sha256", $sha256_expected);
+    write_file("$tmpFile.sha256", $sha256_actual);
 
     if (! -e "$tmpFile.asc") {
         system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
@@ -117,6 +131,60 @@ downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
 downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
 downloadFile("installerScript", "1");
 
+# Upload docker images to dockerhub.
+my $dockerManifest = "";
+my $dockerManifestLatest = "";
+
+for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
+    my $system = $platforms->[0];
+    my $dockerPlatform = $platforms->[1];
+    my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz";
+    downloadFile("dockerImage.$system", "1", $fn);
+
+    print STDERR "loading docker image for $dockerPlatform...\n";
+    system("docker load -i $tmpDir/$fn") == 0 or die;
+
+    my $tag = "nixos/nix:$version-$dockerPlatform";
+    my $latestTag = "nixos/nix:latest-$dockerPlatform";
+
+    print STDERR "tagging $version docker image for $dockerPlatform...\n";
+    system("docker tag nix:$version $tag") == 0 or die;
+
+    if ($isLatest) {
+        print STDERR "tagging latest docker image for $dockerPlatform...\n";
+        system("docker tag nix:$version $latestTag") == 0 or die;
+    }
+
+    print STDERR "pushing $version docker image for $dockerPlatform...\n";
+    system("docker push -q $tag") == 0 or die;
+
+    if ($isLatest) {
+        print STDERR "pushing latest docker image for $dockerPlatform...\n";
+        system("docker push -q $latestTag") == 0 or die;
+    }
+
+    $dockerManifest .= " --amend $tag";
+    $dockerManifestLatest .= " --amend $latestTag"
+}
+
+print STDERR "creating multi-platform docker manifest...\n";
+system("docker manifest rm nixos/nix:$version");
+system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
+if ($isLatest) {
+    print STDERR "creating latest multi-platform docker manifest...\n";
+    system("docker manifest rm nixos/nix:latest");
+    system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
+}
+
+print STDERR "pushing multi-platform docker manifest...\n";
+system("docker manifest push nixos/nix:$version") == 0 or die;
+
+if ($isLatest) {
+    print STDERR "pushing latest multi-platform docker manifest...\n";
+    system("docker manifest push nixos/nix:latest") == 0 or die;
+}
+
+# Upload release files to S3.
 for my $fn (glob "$tmpDir/*") {
     my $name = basename($fn);
     my $dstKey = "$releaseDir/" . $name;

misc/bash/completion.sh

@@ -15,7 +15,7 @@ function _complete_nix {
         else
             COMPREPLY+=("$completion")
         fi
-    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}")
+    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}" 2>/dev/null)
     __ltrim_colon_completions "$cur"
 }
 

misc/systemd/local.mk

@@ -1,7 +1,8 @@
 ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
+  $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644)))
 
-  clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service
+  clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf
 
 endif

misc/systemd/nix-daemon.conf.in

@@ -0,0 +1 @@
+d @localstatedir@/nix/daemon-socket 0755 root root - -

misc/systemd/nix-daemon.service.in

@@ -1,7 +1,9 @@
 [Unit]
 Description=Nix Daemon
+Documentation=man:nix-daemon https://nixos.org/manual
 RequiresMountsFor=@storedir@
 RequiresMountsFor=@localstatedir@
+RequiresMountsFor=@localstatedir@/nix/db
 ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 
 [Service]

misc/zsh/completion.zsh

@@ -4,7 +4,7 @@ function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
   IFS=$'\n'
-  local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]"))
+  local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]" 2>/dev/null))
   IFS="$ifs_bk"
   local tpe="${${res[1]}%%>	*}"
   local -a suggestions

mk/run_test.sh

@@ -14,9 +14,27 @@ if [ -t 1 ]; then
     yellow=""
     normal=""
 fi
-(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+
-log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+run_test () {
-status=$?
+    (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+    log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+    status=$?
+}
+
+run_test "$1"
+
+# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these
+# appear randomly without anyone knowing why.
+# See https://github.com/NixOS/nix/issues/3605 for more info
+if [[ $status -ne 0 && $status -ne 99 && \
+    "$(uname)" == "Darwin" && \
+    "$log" =~ "unexpected EOF reading a line" \
+]]; then
+    echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)"
+    echo "$log" | sed 's/^/    /'
+    run_test "$1"
+fi
+
 if [ $status -eq 0 ]; then
   echo "$post_run_msg [${green}PASS$normal]"
 elif [ $status -eq 99 ]; then

perl/lib/Nix/Store.xs

@@ -240,7 +240,7 @@ SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
             auto h = Hash::parseAny(s, parseHashType(algo));
-            string s = h.to_string(toBase32 ? Base32 : Base16, false);
+            auto s = h.to_string(toBase32 ? Base32 : Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

scripts/check-hydra-status.sh

@@ -14,14 +14,19 @@ curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master
 someBuildFailed=0
 
 for buildId in $BUILDS_FOR_LATEST_EVAL; do
-  buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
+  buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
 
-  buildStatus=$(echo "$buildInfo" | \
+  finished=$(echo "$buildInfo" | jq -r '.finished')
-    jq -r '.buildstatus')
 
-  if [[ "$buildStatus" -ne 0 ]]; then
+  if [[ $finished = 0 ]]; then
+    continue
+  fi
+
+  buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus')
+
+  if [[ $buildStatus != 0 ]]; then
     someBuildFailed=1
-    echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra"
+    echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra: $buildInfo"
   fi
 done
 

scripts/create-darwin-volume.sh

@@ -246,6 +246,7 @@ get_volume_pass() {
 verify_volume_pass() {
     local volume_special="$1" # (i.e., disk1s7)
     local volume_uuid="$2"
+    _sudo "to confirm the password actually unlocks the volume" \
         /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid"
 }
 
@@ -685,11 +686,15 @@ encrypt_volume() {
     local volume_uuid="$1"
     local volume_label="$2"
     local password
+
+    task "Encrypt the Nix volume" >&2
+
     # Note: mount/unmount are late additions to support the right order
     # of operations for creating the volume and then baking its uuid into
     # other artifacts; not as well-trod wrt to potential errors, race
     # conditions, etc.
 
+    _sudo "to mount your Nix volume for encrypting" \
         /usr/sbin/diskutil mount "$volume_label"
 
     password="$(/usr/bin/xxd -l 32 -p -c 256 /dev/random)"
@@ -697,9 +702,10 @@ encrypt_volume() {
         /usr/bin/security -i <<EOF
 add-generic-password -a "$volume_label" -s "$volume_uuid" -l "$volume_label encryption password" -D "Encrypted volume password" -j "Added automatically by the Nix installer for use by $NIX_VOLUME_MOUNTD_DEST" -w "$password" -T /System/Library/CoreServices/APFSUserAgent -T /System/Library/CoreServices/CSUserAgent -T /usr/bin/security "/Library/Keychains/System.keychain"
 EOF
-    builtin printf "%s" "$password" | _sudo "to encrypt your Nix volume" \
+    builtin printf "%s" "$password" | _sudo "to actually encrypt your Nix volume" \
         /usr/sbin/diskutil apfs encryptVolume "$volume_label" -user disk -stdinpassphrase
 
+    _sudo "to unmount the encrypted volume" \
         /usr/sbin/diskutil unmount force "$volume_label"
 }
 

scripts/install-multi-user.sh

@@ -23,10 +23,10 @@ readonly RED='\033[31m'
 # installer allows overriding build user count to speed up installation
 # as creating each user takes non-trivial amount of time on macos
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
-readonly NIX_BUILD_GROUP_ID="30000"
+readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 # darwin installer needs to override these
-NIX_FIRST_BUILD_UID="30001"
+NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
 NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
@@ -609,7 +609,7 @@ EOF
         fi
     fi
     _sudo "to make the basic directory structure of Nix (part 1)" \
-          install -dv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
+          install -dv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool,/daemon-socket} /nix/var/nix/{gcroots,profiles}/per-user
 
     _sudo "to make the basic directory structure of Nix (part 2)" \
           install -dv -g "$NIX_BUILD_GROUP_NAME" -m 1775 /nix/store
@@ -739,7 +739,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              cp -RLp ./store/* "$NIX_ROOT/store/"
+              cp -RPp ./store/* "$NIX_ROOT/store/"
 
         _sudo "to make the new store non-writable at $NIX_ROOT/store" \
               chmod -R ugo-w "$NIX_ROOT/store/"

scripts/install-systemd-multi-user.sh

@@ -9,6 +9,8 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
 readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
 
+readonly TMPFILES_SRC=/lib/tmpfiles.d/nix-daemon.conf
+readonly TMPFILES_DEST=/etc/tmpfiles.d/nix-daemon.conf
 
 # Path for the systemd override unit file to contain the proxy settings
 readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
@@ -83,6 +85,13 @@ EOF
 poly_configure_nix_daemon_service() {
     if [ -e /run/systemd/system ]; then
         task "Setting up the nix-daemon systemd service"
+
+        _sudo "to create the nix-daemon tmpfiles config" \
+              ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST
+
+        _sudo "to run systemd-tmpfiles once to pick that path up" \
+             systemd-tmpfiles --create --prefix=/nix/var/nix
+
         _sudo "to set up the nix-daemon service" \
               systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
 

scripts/install.in

@@ -82,7 +82,7 @@ if [ "$(uname -s)" != "Darwin" ]; then
 fi
 
 if command -v curl > /dev/null 2>&1; then
-    fetch() { curl -L "$1" -o "$2"; }
+    fetch() { curl --fail -L "$1" -o "$2"; }
 elif command -v wget > /dev/null 2>&1; then
     fetch() { wget "$1" -O "$2"; }
 else

scripts/nix-profile.sh.in

@@ -24,6 +24,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 
+    # Only use MANPATH if it is already set. In general `man` will just simply
+    # pick up `.nix-profile/share/man` because is it close to `.nix-profile/bin`
+    # which is in the $PATH. For more info, run `manpath -d`.
     if [ -n "${MANPATH-}" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi

src/build-remote/build-remote.cc

@@ -14,6 +14,7 @@
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "serialise.hh"
+#include "build-result.hh"
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
@@ -32,7 +33,7 @@ std::string escapeUri(std::string uri)
     return uri;
 }
 
-static string currentLoad;
+static std::string currentLoad;
 
 static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
@@ -97,7 +98,7 @@ static int main_build_remote(int argc, char * * argv)
         }
 
         std::optional<StorePath> drvPath;
-        string storeUri;
+        std::string storeUri;
 
         while (true) {
 
@@ -183,7 +184,7 @@ static int main_build_remote(int argc, char * * argv)
                     else
                     {
                         // build the hint template.
-                        string errorText =
+                        std::string errorText =
                             "Failed to find a machine for remote build!\n"
                             "derivation: %s\nrequired (system, features): (%s, %s)";
                         errorText += "\n%s available machines:";
@@ -193,7 +194,7 @@ static int main_build_remote(int argc, char * * argv)
                             errorText += "\n(%s, %s, %s, %s)";
 
                         // add the template values.
-                        string drvstr;
+                        std::string drvstr;
                         if (drvPath.has_value())
                             drvstr = drvPath->to_string();
                         else
@@ -208,7 +209,7 @@ static int main_build_remote(int argc, char * * argv)
 
                         for (auto & m : machines)
                             error
-                                % concatStringsSep<vector<string>>(", ", m.systemTypes)
+                                % concatStringsSep<std::vector<std::string>>(", ", m.systemTypes)
                                 % m.maxJobs
                                 % concatStringsSep<StringSet>(", ", m.supportedFeatures)
                                 % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
@@ -299,7 +300,7 @@ connected:
 
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
             for (auto & outputName : wantedOutputs) {
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };

src/libcmd/command.cc

@@ -209,7 +209,7 @@ void BuiltPathsCommand::run(ref<Store> store)
         for (auto & p : store->queryAllValidPaths())
             paths.push_back(BuiltPath::Opaque{p});
     } else {
-        paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
+        paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations
@@ -260,7 +260,8 @@ Strings editorFor(const Pos & pos)
     if (pos.line > 0 && (
         editor.find("emacs") != std::string::npos ||
         editor.find("nano") != std::string::npos ||
-        editor.find("vim") != std::string::npos))
+        editor.find("vim") != std::string::npos ||
+        editor.find("kak") != std::string::npos))
         args.push_back(fmt("+%d", pos.line));
     args.push_back(pos.file);
     return args;

src/libcmd/command.hh

@@ -5,7 +5,6 @@
 #include "common-eval-args.hh"
 #include "path.hh"
 #include "flake/lockfile.hh"
-#include "store-api.hh"
 
 #include <optional>
 
@@ -84,14 +83,6 @@ struct MixFlakeOptions : virtual Args, EvalCommand
     { return {}; }
 };
 
-/* How to handle derivations in commands that operate on store paths. */
-enum class OperateOn {
-    /* Operate on the output path. */
-    Output,
-    /* Operate on the .drv path. */
-    Derivation
-};
-
 struct SourceExprCommand : virtual Args, MixFlakeOptions
 {
     std::optional<Path> file;
@@ -115,19 +106,6 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
     void completeInstallable(std::string_view prefix);
 };
 
-enum class Realise {
-    /* Build the derivation. Postcondition: the
-       derivation outputs exist. */
-    Outputs,
-    /* Don't build the derivation. Postcondition: the store derivation
-       exists. */
-    Derivation,
-    /* Evaluate in dry-run mode. Postcondition: nothing. */
-    // FIXME: currently unused, but could be revived if we can
-    // evaluate derivations in-memory.
-    Nothing
-};
-
 /* A command that operates on a list of "installables", which can be
    store paths, attribute paths, Nix expressions, etc. */
 struct InstallablesCommand : virtual Args, SourceExprCommand
@@ -240,38 +218,6 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-BuiltPaths build(
-    ref<Store> evalStore,
-    ref<Store> store, Realise mode,
-    const std::vector<std::shared_ptr<Installable>> & installables,
-    BuildMode bMode = bmNormal);
-
-std::set<StorePath> toStorePaths(
-    ref<Store> evalStore,
-    ref<Store> store,
-    Realise mode,
-    OperateOn operateOn,
-    const std::vector<std::shared_ptr<Installable>> & installables);
-
-StorePath toStorePath(
-    ref<Store> evalStore,
-    ref<Store> store,
-    Realise mode,
-    OperateOn operateOn,
-    std::shared_ptr<Installable> installable);
-
-std::set<StorePath> toDerivations(
-    ref<Store> store,
-    const std::vector<std::shared_ptr<Installable>> & installables,
-    bool useDeriver = false);
-
-BuiltPaths toBuiltPaths(
-    ref<Store> evalStore,
-    ref<Store> store,
-    Realise mode,
-    OperateOn operateOn,
-    const std::vector<std::shared_ptr<Installable>> & installables);
-
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */
 Strings editorFor(const Pos & pos);

src/libcmd/installables.cc

@@ -12,6 +12,7 @@
 #include "eval-cache.hh"
 #include "url.hh"
 #include "registry.hh"
+#include "build-result.hh"
 
 #include <regex>
 #include <queue>
@@ -97,7 +98,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath(".")));
+                parseFlakeRef(flakeRef, absPath("."), true));
         }}
     });
 
@@ -133,7 +134,9 @@ SourceExprCommand::SourceExprCommand()
     addFlag({
         .longName = "file",
         .shortName = 'f',
-        .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.",
+        .description =
+            "Interpret installables as attribute paths relative to the Nix expression stored in *file*. "
+            "If *file* is the character -, then a Nix expression will be read from standard input.",
         .category = installablesCategory,
         .labels = {"file"},
         .handler = {&file},
@@ -158,7 +161,10 @@ SourceExprCommand::SourceExprCommand()
 
 Strings SourceExprCommand::getDefaultFlakeAttrPaths()
 {
-    return {"defaultPackage." + settings.thisSystem.get()};
+    return {
+        "packages." + settings.thisSystem.get() + ".default",
+        "defaultPackage." + settings.thisSystem.get()
+    };
 }
 
 Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
@@ -269,9 +275,9 @@ void completeFlakeRefWithFragment(
                 auto attr = root->findAlongAttrPath(attrPath);
                 if (!attr) continue;
 
-                for (auto & attr2 : attr->getAttrs()) {
+                for (auto & attr2 : (*attr)->getAttrs()) {
                     if (hasPrefix(attr2, lastAttr)) {
-                        auto attrPath2 = attr->getAttrPath(attr2);
+                        auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
                         completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
@@ -465,11 +471,10 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
 
     std::vector<DerivationInfo> res;
     for (auto & drvInfo : drvInfos) {
-        res.push_back({
+        auto drvPath = drvInfo.queryDrvPath();
-            state->store->parseStorePath(drvInfo.queryDrvPath()),
+        if (!drvPath)
-            state->store->maybeParseStorePath(drvInfo.queryOutPath()),
+            throw Error("'%s' is not a derivation", what());
-            drvInfo.queryOutputName()
+        res.push_back({ *drvPath, drvInfo.queryOutputName() });
-        });
     }
 
     return res;
@@ -545,13 +550,14 @@ InstallableFlake::InstallableFlake(
     SourceExprCommand * cmd,
     ref<EvalState> state,
     FlakeRef && flakeRef,
-    Strings && attrPaths,
+    std::string_view fragment,
-    Strings && prefixes,
+    Strings attrPaths,
+    Strings prefixes,
     const flake::LockFlags & lockFlags)
     : InstallableValue(state),
       flakeRef(flakeRef),
-      attrPaths(attrPaths),
+      attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
-      prefixes(prefixes),
+      prefixes(fragment == "" ? Strings{} : prefixes),
       lockFlags(lockFlags)
 {
     if (cmd && cmd->getAutoArgs(*state)->size())
@@ -565,29 +571,37 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto cache = openEvalCache(*state, lockedFlake);
     auto root = cache->getRoot();
 
+    Suggestions suggestions;
+
     for (auto & attrPath : getActualAttrPaths()) {
-        auto attr = root->findAlongAttrPath(
+        debug("trying flake output attribute '%s'", attrPath);
+
+        auto attrOrSuggestions = root->findAlongAttrPath(
             parseAttrPath(*state, attrPath),
             true
         );
 
-        if (!attr) continue;
+        if (!attrOrSuggestions) {
+            suggestions += attrOrSuggestions.getSuggestions();
+            continue;
+        }
+
+        auto attr = *attrOrSuggestions;
 
         if (!attr->isDerivation())
             throw Error("flake output attribute '%s' is not a derivation", attrPath);
 
         auto drvPath = attr->forceDerivation();
 
-        auto drvInfo = DerivationInfo{
+        auto drvInfo = DerivationInfo {
             std::move(drvPath),
-            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
             attr->getAttr(state->sOutputName)->getString()
         };
 
         return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
     }
 
-    throw Error("flake '%s' does not provide attribute %s",
+    throw Error(suggestions, "flake '%s' does not provide attribute %s",
         flakeRef, showAttrPaths(getActualAttrPaths()));
 }
 
@@ -606,17 +620,24 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 
     auto emptyArgs = state.allocBindings(0);
 
+    Suggestions suggestions;
+
     for (auto & attrPath : getActualAttrPaths()) {
         try {
             auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
             state.forceValue(*v, pos);
             return {v, pos};
         } catch (AttrPathNotFound & e) {
+            suggestions += e.info().suggestions;
         }
     }
 
-    throw Error("flake '%s' does not provide attribute %s",
+    throw Error(
-        flakeRef, showAttrPaths(getActualAttrPaths()));
+        suggestions,
+        "flake '%s' does not provide attribute %s",
+        flakeRef,
+        showAttrPaths(getActualAttrPaths())
+    );
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
@@ -631,7 +652,7 @@ InstallableFlake::getCursors(EvalState & state)
 
     for (auto & attrPath : getActualAttrPaths()) {
         auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
-        if (attr) res.push_back({attr, attrPath});
+        if (attr) res.push_back({*attr, attrPath});
     }
 
     return res;
@@ -676,7 +697,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         auto state = getEvalState();
         auto vFile = state->allocValue();
 
-        if (file)
+        if (file == "-") {
+            auto e = state->parseStdin();
+            state->eval(e, *vFile);
+        } else if (file)
             state->evalFile(lookupFileArg(*state, *file), *vFile);
         else {
             auto e = state->parseExprFromString(*expr, absPath("."));
@@ -708,7 +732,8 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
                         this,
                         getEvalState(),
                         std::move(flakeRef),
-                        fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
+                        fragment,
+                        getDefaultFlakeAttrPaths(),
                         getDefaultFlakeAttrPathPrefixes(),
                         lockFlags));
                 continue;
@@ -731,56 +756,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
+BuiltPaths Installable::build(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode)
 {
     BuiltPaths res;
-    for (const auto & b : hopefullyBuiltPaths)
+    for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
-        std::visit(
+        res.push_back(builtPath);
-            overloaded{
-                [&](const DerivedPath::Opaque & bo) {
-                    res.push_back(BuiltPath::Opaque{bo.path});
-                },
-                [&](const DerivedPath::Built & bfd) {
-                    OutputPathMap outputs;
-                    auto drv = evalStore->readDerivation(bfd.drvPath);
-                    auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
-                    auto drvOutputs = drv.outputsAndOptPaths(*store);
-                    for (auto & output : bfd.outputs) {
-                        if (!outputHashes.count(output))
-                            throw Error(
-                                "the derivation '%s' doesn't have an output named '%s'",
-                                store->printStorePath(bfd.drvPath), output);
-                        if (settings.isExperimentalFeatureEnabled(
-                                Xp::CaDerivations)) {
-                            auto outputId =
-                                DrvOutput{outputHashes.at(output), output};
-                            auto realisation =
-                                store->queryRealisation(outputId);
-                            if (!realisation)
-                                throw Error(
-                                    "cannot operate on an output of unbuilt "
-                                    "content-addressed derivation '%s'",
-                                    outputId.to_string());
-                            outputs.insert_or_assign(
-                                output, realisation->outPath);
-                        } else {
-                            // If ca-derivations isn't enabled, assume that
-                            // the output path is statically known.
-                            assert(drvOutputs.count(output));
-                            assert(drvOutputs.at(output).second);
-                            outputs.insert_or_assign(
-                                output, *drvOutputs.at(output).second);
-                        }
-                    }
-                    res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
-                },
-            },
-            b.raw());
-
     return res;
 }
 
-BuiltPaths build(
+std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -791,21 +780,96 @@ BuiltPaths build(
         settings.readOnlyMode = true;
 
     std::vector<DerivedPath> pathsToBuild;
+    std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap;
 
     for (auto & i : installables) {
-        auto b = i->toDerivedPaths();
+        for (auto b : i->toDerivedPaths()) {
-        pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
+            pathsToBuild.push_back(b);
+            backmap[b].push_back(i);
+        }
     }
 
-    if (mode == Realise::Nothing || mode == Realise::Derivation)
+    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
-        printMissing(store, pathsToBuild, lvlError);
-    else if (mode == Realise::Outputs)
-        store->buildPaths(pathsToBuild, bMode, evalStore);
 
-    return getBuiltPaths(evalStore, store, pathsToBuild);
+    switch (mode) {
+
+    case Realise::Nothing:
+    case Realise::Derivation:
+        printMissing(store, pathsToBuild, lvlError);
+
+        for (auto & path : pathsToBuild) {
+            for (auto & installable : backmap[path]) {
+                std::visit(overloaded {
+                    [&](const DerivedPath::Built & bfd) {
+                        OutputPathMap outputs;
+                        auto drv = evalStore->readDerivation(bfd.drvPath);
+                        auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
+                        auto drvOutputs = drv.outputsAndOptPaths(*store);
+                        for (auto & output : bfd.outputs) {
+                            if (!outputHashes.count(output))
+                                throw Error(
+                                    "the derivation '%s' doesn't have an output named '%s'",
+                                    store->printStorePath(bfd.drvPath), output);
+                            if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+                                DrvOutput outputId { outputHashes.at(output), output };
+                                auto realisation = store->queryRealisation(outputId);
+                                if (!realisation)
+                                    throw Error(
+                                        "cannot operate on an output of unbuilt "
+                                        "content-addressed derivation '%s'",
+                                        outputId.to_string());
+                                outputs.insert_or_assign(output, realisation->outPath);
+                            } else {
+                                // If ca-derivations isn't enabled, assume that
+                                // the output path is statically known.
+                                assert(drvOutputs.count(output));
+                                assert(drvOutputs.at(output).second);
+                                outputs.insert_or_assign(
+                                    output, *drvOutputs.at(output).second);
+                            }
+                        }
+                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                    },
+                    [&](const DerivedPath::Opaque & bo) {
+                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                    },
+                }, path.raw());
+            }
+        }
+
+        break;
+
+    case Realise::Outputs: {
+        for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
+            if (!buildResult.success())
+                buildResult.rethrow();
+
+            for (auto & installable : backmap[buildResult.path]) {
+                std::visit(overloaded {
+                    [&](const DerivedPath::Built & bfd) {
+                        std::map<std::string, StorePath> outputs;
+                        for (auto & path : buildResult.builtOutputs)
+                            outputs.emplace(path.first.outputName, path.second.outPath);
+                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                    },
+                    [&](const DerivedPath::Opaque & bo) {
+                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                    },
+                }, buildResult.path.raw());
+            }
+        }
+
+        break;
+    }
+
+    default:
+        assert(false);
+    }
+
+    return res;
 }
 
-BuiltPaths toBuiltPaths(
+BuiltPaths Installable::toBuiltPaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -813,19 +877,19 @@ BuiltPaths toBuiltPaths(
     const std::vector<std::shared_ptr<Installable>> & installables)
 {
     if (operateOn == OperateOn::Output)
-        return build(evalStore, store, mode, installables);
+        return Installable::build(evalStore, store, mode, installables);
     else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
         BuiltPaths res;
-        for (auto & drvPath : toDerivations(store, installables, true))
+        for (auto & drvPath : Installable::toDerivations(store, installables, true))
             res.push_back(BuiltPath::Opaque{drvPath});
         return res;
     }
 }
 
-StorePathSet toStorePaths(
+StorePathSet Installable::toStorePaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -839,7 +903,7 @@ StorePathSet toStorePaths(
     return outPaths;
 }
 
-StorePath toStorePath(
+StorePath Installable::toStorePath(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -853,7 +917,7 @@ StorePath toStorePath(
     return *paths.begin();
 }
 
-StorePathSet toDerivations(
+StorePathSet Installable::toDerivations(
     ref<Store> store,
     const std::vector<std::shared_ptr<Installable>> & installables,
     bool useDeriver)

src/libcmd/installables.hh

@@ -5,6 +5,7 @@
 #include "path-with-outputs.hh"
 #include "derived-path.hh"
 #include "eval.hh"
+#include "store-api.hh"
 #include "flake/flake.hh"
 
 #include <optional>
@@ -29,6 +30,27 @@ struct UnresolvedApp
     App resolve(ref<Store> evalStore, ref<Store> store);
 };
 
+enum class Realise {
+    /* Build the derivation. Postcondition: the
+       derivation outputs exist. */
+    Outputs,
+    /* Don't build the derivation. Postcondition: the store derivation
+       exists. */
+    Derivation,
+    /* Evaluate in dry-run mode. Postcondition: nothing. */
+    // FIXME: currently unused, but could be revived if we can
+    // evaluate derivations in-memory.
+    Nothing
+};
+
+/* How to handle derivations in commands that operate on store paths. */
+enum class OperateOn {
+    /* Operate on the output path. */
+    Output,
+    /* Operate on the .drv path. */
+    Derivation
+};
+
 struct Installable
 {
     virtual ~Installable() { }
@@ -68,6 +90,46 @@ struct Installable
     {
         return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
     }
+
+    static BuiltPaths build(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        const std::vector<std::shared_ptr<Installable>> & installables,
+        BuildMode bMode = bmNormal);
+
+    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        const std::vector<std::shared_ptr<Installable>> & installables,
+        BuildMode bMode = bmNormal);
+
+    static std::set<StorePath> toStorePaths(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const std::vector<std::shared_ptr<Installable>> & installables);
+
+    static StorePath toStorePath(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        std::shared_ptr<Installable> installable);
+
+    static std::set<StorePath> toDerivations(
+        ref<Store> store,
+        const std::vector<std::shared_ptr<Installable>> & installables,
+        bool useDeriver = false);
+
+    static BuiltPaths toBuiltPaths(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const std::vector<std::shared_ptr<Installable>> & installables);
 };
 
 struct InstallableValue : Installable
@@ -79,7 +141,6 @@ struct InstallableValue : Installable
     struct DerivationInfo
     {
         StorePath drvPath;
-        std::optional<StorePath> outPath;
         std::string outputName;
     };
 
@@ -102,8 +163,9 @@ struct InstallableFlake : InstallableValue
         SourceExprCommand * cmd,
         ref<EvalState> state,
         FlakeRef && flakeRef,
-        Strings && attrPaths,
+        std::string_view fragment,
-        Strings && prefixes,
+        Strings attrPaths,
+        Strings prefixes,
         const flake::LockFlags & lockFlags);
 
     std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }

src/libcmd/repl.cc

@@ -25,6 +25,7 @@ extern "C" {
 #include "eval-inline.hh"
 #include "attr-path.hh"
 #include "store-api.hh"
+#include "log-store.hh"
 #include "common-eval-args.hh"
 #include "get-drvs.hh"
 #include "derivations.hh"
@@ -45,7 +46,7 @@ struct NixRepl
     : gc
     #endif
 {
-    string curDir;
+    std::string curDir;
     ref<EvalState> state;
     Bindings * autoArgs;
 
@@ -65,10 +66,10 @@ struct NixRepl
     NixRepl(ref<EvalState> state);
     ~NixRepl();
     void mainLoop(const std::vector<std::string> & files);
-    StringSet completePrefix(string prefix);
+    StringSet completePrefix(const std::string & prefix);
-    bool getLine(string & input, const std::string &prompt);
+    bool getLine(std::string & input, const std::string &prompt);
     StorePath getDerivationPath(Value & v);
-    bool processLine(string line);
+    bool processLine(std::string line);
     void loadFile(const Path & path);
     void loadFlake(const std::string & flakeRef);
     void initEnv();
@@ -76,22 +77,21 @@ struct NixRepl
     void reloadFiles();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol & name, Value & v);
-    Expr * parseString(string s);
+    Expr * parseString(std::string s);
-    void evalString(string s, Value & v);
+    void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace &dt);
 
-    typedef set<Value *> ValuesSeen;
+    typedef std::set<Value *> ValuesSeen;
     std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth);
     std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen);
 };
 
 
-string removeWhitespace(string s)
+std::string removeWhitespace(std::string s)
 {
     s = chomp(s);
     size_t n = s.find_first_not_of(" \n\r\t");
-    if (n != string::npos) s = string(s, n);
+    if (n != std::string::npos) s = std::string(s, n);
-
     return s;
 }
 
@@ -111,7 +111,7 @@ NixRepl::~NixRepl()
     write_history(historyFile.c_str());
 }
 
-string runNix(Path program, const Strings & args,
+std::string runNix(Path program, const Strings & args,
     const std::optional<std::string> & input = {})
 {
     auto subprocessEnv = getEnv();
@@ -229,7 +229,7 @@ std::ostream& showDebugTrace(std::ostream &out, const DebugTrace &dt)
 
 void NixRepl::mainLoop(const std::vector<std::string> & files)
 {
-    string error = ANSI_RED "error:" ANSI_NORMAL " ";
+    std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
     notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n");
 
     if (!files.empty()) {
@@ -297,7 +297,7 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
 }
 
 
-bool NixRepl::getLine(string & input, const std::string &prompt)
+bool NixRepl::getLine(std::string & input, const std::string & prompt)
 {
     struct sigaction act, old;
     sigset_t savedSignalMask, set;
@@ -342,7 +342,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
 }
 
 
-StringSet NixRepl::completePrefix(string prefix)
+StringSet NixRepl::completePrefix(const std::string & prefix)
 {
     StringSet completions;
 
@@ -358,7 +358,7 @@ StringSet NixRepl::completePrefix(string prefix)
 
     size_t slash, dot;
 
-    if ((slash = cur.rfind('/')) != string::npos) {
+    if ((slash = cur.rfind('/')) != std::string::npos) {
         try {
             auto dir = std::string(cur, 0, slash);
             auto prefix2 = std::string(cur, slash + 1);
@@ -368,11 +368,11 @@ StringSet NixRepl::completePrefix(string prefix)
             }
         } catch (Error &) {
         }
-    } else if ((dot = cur.rfind('.')) == string::npos) {
+    } else if ((dot = cur.rfind('.')) == std::string::npos) {
         /* This is a variable name; look it up in the current scope. */
         StringSet::iterator i = varNames.lower_bound(cur);
         while (i != varNames.end()) {
-            if (string(*i, 0, cur.size()) != cur) break;
+            if (i->substr(0, cur.size()) != cur) break;
             completions.insert(prev + *i);
             i++;
         }
@@ -381,8 +381,8 @@ StringSet NixRepl::completePrefix(string prefix)
             /* This is an expression that should evaluate to an
                attribute set.  Evaluate it to get the names of the
                attributes. */
-            string expr(cur, 0, dot);
+            auto expr = cur.substr(0, dot);
-            string cur2 = string(cur, dot + 1);
+            auto cur2 = cur.substr(dot + 1);
 
             Expr * e = parseString(expr);
             Value v;
@@ -390,8 +390,8 @@ StringSet NixRepl::completePrefix(string prefix)
             state->forceAttrs(v, noPos);
 
             for (auto & i : *v.attrs) {
-                string name = i.name;
+                std::string name = i.name;
-                if (string(name, 0, cur2.size()) != cur2) continue;
+                if (name.substr(0, cur2.size()) != cur2) continue;
                 completions.insert(prev + expr + "." + name);
             }
 
@@ -410,7 +410,7 @@ StringSet NixRepl::completePrefix(string prefix)
 }
 
 
-bool isVarName(const string & s)
+static bool isVarName(std::string_view s)
 {
     if (s.size() == 0) return false;
     char c = s[0];
@@ -429,13 +429,12 @@ StorePath NixRepl::getDerivationPath(Value & v) {
     auto drvInfo = getDerivation(*state, v, false);
     if (!drvInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
-    Path drvPathRaw = drvInfo->queryDrvPath();
+    auto drvPath = drvInfo->queryDrvPath();
-    if (drvPathRaw == "")
+    if (!drvPath)
-        throw Error("expression did not evaluate to a valid derivation (no drv path)");
+        throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)");
-    StorePath drvPath = state->store->parseStorePath(drvPathRaw);
+    if (!state->store->isValidPath(*drvPath))
-    if (!state->store->isValidPath(drvPath))
+        throw Error("expression evaluated to invalid derivation '%s'", state->store->printStorePath(*drvPath));
-        throw Error("expression did not evaluate to a valid derivation (invalid drv path)");
+    return *drvPath;
-    return drvPath;
 }
 
 void NixRepl::loadDebugTraceEnv(DebugTrace &dt)
@@ -457,18 +456,19 @@ void NixRepl::loadDebugTraceEnv(DebugTrace &dt)
     }
 }
 
-bool NixRepl::processLine(string line)
+bool NixRepl::processLine(std::string line)
 {
+    line = trim(line);
     if (line == "") return true;
 
     _isInterrupted = false;
 
-    string command, arg;
+    std::string command, arg;
 
     if (line[0] == ':') {
         size_t p = line.find_first_of(" \n\r\t");
-        command = string(line, 0, p);
+        command = line.substr(0, p);
-        if (p != string::npos) arg = removeWhitespace(string(line, p));
+        if (p != std::string::npos) arg = removeWhitespace(line.substr(p));
     } else {
         arg = line;
     }
@@ -667,9 +667,16 @@ bool NixRepl::processLine(string line)
             bool foundLog = false;
             RunPager pager;
             for (auto & sub : subs) {
-                auto log = sub->getBuildLog(drvPath);
+                auto * logSubP = dynamic_cast<LogStore *>(&*sub);
+                if (!logSubP) {
+                    printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri());
+                    continue;
+                }
+                auto & logSub = *logSubP;
+
+                auto log = logSub.getBuildLog(drvPath);
                 if (log) {
-                    printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri());
+                    printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri());
                     logger->writeToStdout(*log);
                     foundLog = true;
                     break;
@@ -733,13 +740,13 @@ bool NixRepl::processLine(string line)
 
     else {
         size_t p = line.find('=');
-        string name;
+        std::string name;
-        if (p != string::npos &&
+        if (p != std::string::npos &&
             p < line.size() &&
             line[p + 1] != '=' &&
-            isVarName(name = removeWhitespace(string(line, 0, p))))
+            isVarName(name = removeWhitespace(line.substr(0, p))))
         {
-            Expr * e = parseString(string(line, p + 1));
+            Expr * e = parseString(line.substr(p + 1));
             Value & v(*state->allocValue());
             v.mkThunk(env, e);
             addVarToScope(state->symbols.create(name), v);
@@ -766,9 +773,12 @@ void NixRepl::loadFile(const Path & path)
 
 void NixRepl::loadFlake(const std::string & flakeRefS)
 {
+    if (flakeRefS.empty())
+        throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)");
+
     auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true);
-    if (evalSettings.pureEval && !flakeRef.input.isImmutable())
+    if (evalSettings.pureEval && !flakeRef.input.isLocked())
-        throw Error("cannot use ':load-flake' on mutable flake reference '%s' (use --impure to override)", flakeRefS);
+        throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
 
     Value v;
 
@@ -829,7 +839,7 @@ void NixRepl::addAttrsToScope(Value & attrs)
     for (auto & i : *attrs.attrs) {
         staticEnv->vars.emplace_back(i.name, displ);
         env->values[displ++] = i.value;
-        varNames.insert((string) i.name);
+        varNames.insert((std::string) i.name);
     }
     staticEnv->sort();
     staticEnv->deduplicate();
@@ -846,17 +856,18 @@ void NixRepl::addVarToScope(const Symbol & name, Value & v)
     staticEnv->vars.emplace_back(name, displ);
     staticEnv->sort();
     env->values[displ++] = &v;
-    varNames.insert((string) name);
+    varNames.insert((std::string) name);
 }
 
-Expr * NixRepl::parseString(string s)
+
+Expr * NixRepl::parseString(std::string s)
 {
     Expr * e = state->parseExprFromString(std::move(s), curDir, staticEnv);
     return e;
 }
 
 
-void NixRepl::evalString(string s, Value & v)
+void NixRepl::evalString(std::string s, Value & v)
 {
     Expr * e = parseString(s);
     e->eval(*state, *env, v);
@@ -925,14 +936,17 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
             str << "«derivation ";
             Bindings::iterator i = v.attrs->find(state->sDrvPath);
             PathSet context;
-            Path drvPath = i != v.attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "???";
+            if (i != v.attrs->end())
-            str << drvPath << "»";
+                str << state->store->printStorePath(state->coerceToStorePath(*i->pos, *i->value, context));
+            else
+                str << "???";
+            str << "»";
         }
 
         else if (maxDepth > 0) {
             str << "{ ";
 
-            typedef std::map<string, Value *> Sorted;
+            typedef std::map<std::string, Value *> Sorted;
             Sorted sorted;
             for (auto & i : *v.attrs)
                 sorted[i.name] = i.value;
@@ -943,7 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                 else
                     printStringValue(str, i.first.c_str());
                 str << " = ";
-                if (seen.find(i.second) != seen.end())
+                if (seen.count(i.second))
                     str << "«repeated»";
                 else
                     try {

src/libexpr/attr-path.cc

@@ -9,7 +9,7 @@ namespace nix {
 static Strings parseAttrPath(std::string_view s)
 {
     Strings res;
-    string cur;
+    std::string cur;
     auto i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
@@ -41,7 +41,7 @@ std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
 }
 
 
-std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const std::string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
@@ -74,8 +74,14 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
-            if (a == v->attrs->end())
+            if (a == v->attrs->end()) {
-                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
+                std::set<std::string> attrNames;
+                for (auto & attr : *v->attrs)
+                    attrNames.insert(attr.name);
+
+                auto suggestions = Suggestions::bestMatches(attrNames, attr);
+                throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);
+            }
             v = &*a->value;
             pos = *a->pos;
         }
@@ -121,7 +127,7 @@ Pos findPackageFilename(EvalState & state, Value & v, std::string what)
     std::string filename(pos, 0, colon);
     unsigned int lineno;
     try {
-        lineno = std::stoi(std::string(pos, colon + 1, string::npos));
+        lineno = std::stoi(std::string(pos, colon + 1, std::string::npos));
     } catch (std::invalid_argument & e) {
         throw ParseError("cannot parse line number '%s'", pos);
     }

src/libexpr/attr-path.hh

@@ -10,8 +10,11 @@ namespace nix {
 MakeError(AttrPathNotFound, Error);
 MakeError(NoPositionInfo, Error);
 
-std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(
-    Bindings & autoArgs, Value & vIn);
+    EvalState & state,
+    const std::string & attrPath,
+    Bindings & autoArgs,
+    Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findPackageFilename(EvalState & state, Value & v, std::string what);

src/libexpr/attr-set.hh

@@ -105,7 +105,7 @@ public:
         for (size_t n = 0; n < size_; n++)
             res.emplace_back(&attrs[n]);
         std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) {
-            return (const string &) a->name < (const string &) b->name;
+            return (const std::string &) a->name < (const std::string &) b->name;
         });
         return res;
     }

src/libexpr/common-eval-args.cc

@@ -77,7 +77,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
     for (auto & i : autoArgs) {
         auto v = state.allocValue();
         if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
+            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath(".")));
         else
             v->mkString(((std::string_view) i.second).substr(1));
         res.insert(state.symbols.create(i.first), v);
@@ -85,17 +85,17 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
     return res.finish();
 }
 
-Path lookupFileArg(EvalState & state, string s)
+Path lookupFileArg(EvalState & state, std::string_view s)
 {
     if (isUri(s)) {
         return state.store->toRealPath(
             fetchers::downloadTarball(
                 state.store, resolveUri(s), "source", false).first.storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
-        Path p = s.substr(1, s.size() - 2);
+        Path p(s.substr(1, s.size() - 2));
         return state.findFile(p);
     } else
-        return absPath(s);
+        return absPath(std::string(s));
 }
 
 }

src/libexpr/common-eval-args.hh

@@ -22,6 +22,6 @@ private:
     std::map<std::string, std::string> autoArgs;
 };
 
-Path lookupFileArg(EvalState & state, string s);
+Path lookupFileArg(EvalState & state, std::string_view s);
 
 }

src/libexpr/eval-cache.cc

@@ -21,6 +21,8 @@ struct AttrDb
 {
     std::atomic_bool failed{false};
 
+    const Store & cfg;
+
     struct State
     {
         SQLite db;
@@ -33,8 +35,9 @@ struct AttrDb
 
     std::unique_ptr<Sync<State>> _state;
 
-    AttrDb(const Hash & fingerprint)
+    AttrDb(const Store & cfg, const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
+        : cfg(cfg)
+        , _state(std::make_unique<Sync<State>>())
     {
         auto state(_state->lock());
 
@@ -254,10 +257,10 @@ struct AttrDb
                 return {{rowId, attrs}};
             }
             case AttrType::String: {
-                std::vector<std::pair<Path, std::string>> context;
+                NixStringContext context;
                 if (!queryAttribute.isNull(3))
                     for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(s));
+                        context.push_back(decodeContext(cfg, s));
                 return {{rowId, string_t{queryAttribute.getStr(2), context}}};
             }
             case AttrType::Bool:
@@ -274,10 +277,10 @@ struct AttrDb
     }
 };
 
-static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+static std::shared_ptr<AttrDb> makeAttrDb(const Store & cfg, const Hash & fingerprint)
 {
     try {
-        return std::make_shared<AttrDb>(fingerprint);
+        return std::make_shared<AttrDb>(cfg, fingerprint);
     } catch (SQLiteError &) {
         ignoreException();
         return nullptr;
@@ -288,7 +291,7 @@ EvalCache::EvalCache(
     std::optional<std::reference_wrapper<const Hash>> useCache,
     EvalState & state,
     RootLoader rootLoader)
-    : db(useCache ? makeAttrDb(*useCache) : nullptr)
+    : db(useCache ? makeAttrDb(*state.store, *useCache) : nullptr)
     , state(state)
     , rootLoader(rootLoader)
 {
@@ -406,6 +409,16 @@ Value & AttrCursor::forceValue()
     return v;
 }
 
+Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
+{
+    auto attrNames = getAttrs();
+    std::set<std::string> strAttrNames;
+    for (auto & name : attrNames)
+        strAttrNames.insert(std::string(name));
+
+    return Suggestions::bestMatches(strAttrNames, name);
+}
+
 std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
 {
     if (root->db) {
@@ -446,6 +459,11 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
         return nullptr;
         //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
+    for (auto & attr : *v.attrs) {
+        if (root->db)
+            root->db->setPlaceholder({cachedValue->first, attr.name});
+    }
+
     auto attr = v.attrs->get(name);
 
     if (!attr) {
@@ -464,7 +482,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
         cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
     }
 
-    return std::make_shared<AttrCursor>(
+    return make_ref<AttrCursor>(
         root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
 }
 
@@ -473,27 +491,31 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
 {
     auto p = maybeGetAttr(name, forceErrors);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
-    return p;
+    return ref(p);
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
 {
     return getAttr(root->state.symbols.create(name));
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        res = res->maybeGetAttr(attr, force);
+        auto child = res->maybeGetAttr(attr, force);
-        if (!res) return {};
+        if (!child) {
+            auto suggestions = res->getSuggestionsForAttr(attr);
+            return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
         }
-    return res;
+        res = child;
+    }
+    return ref(res);
 }
 
 std::string AttrCursor::getString()
@@ -527,7 +549,7 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+                    if (!root->state.store->isValidPath(c.first)) {
                         valid = false;
                         break;
                     }
@@ -544,7 +566,7 @@ string_t AttrCursor::getStringWithContext()
     auto & v = forceValue();
 
     if (v.type() == nString)
-        return {v.string.s, v.getContext()};
+        return {v.string.s, v.getContext(*root->state.store)};
     else if (v.type() == nPath)
         return {v.path, {}};
     else
@@ -599,7 +621,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
     for (auto & attr : *getValue().attrs)
         attrs.push_back(attr.name);
     std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
-        return (const string &) a < (const string &) b;
+        return (const std::string &) a < (const std::string &) b;
     });
 
     if (root->db)

src/libexpr/eval-cache.hh

@@ -52,7 +52,7 @@ struct misc_t {};
 struct failed_t {};
 typedef uint64_t AttrId;
 typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+typedef std::pair<std::string, NixStringContext> string_t;
 
 typedef std::variant<
     std::vector<Symbol>,
@@ -94,15 +94,17 @@ public:
 
     std::string getAttrPathStr(Symbol name) const;
 
+    Suggestions getSuggestionsForAttr(Symbol name);
+
     std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+    ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
 
-    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+    ref<AttrCursor> getAttr(std::string_view name);
 
-    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
 
     std::string getString();
 

src/libexpr/eval-inline.hh

@@ -38,6 +38,81 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 }
 
 
+/* Note: Various places expect the allocated memory to be zeroed. */
+[[gnu::always_inline]]
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_MALLOC(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
+
+[[gnu::always_inline]]
+Value * EvalState::allocValue()
+{
+#if HAVE_BOEHMGC
+    /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
+       GC_malloc_many returns a linked list of objects of the given size, where the first word
+       of each object is also the pointer to the next object in the list. This also means that we
+       have to explicitly clear the first word of every object we take. */
+    if (!*valueAllocCache) {
+        *valueAllocCache = GC_malloc_many(sizeof(Value));
+        if (!*valueAllocCache) throw std::bad_alloc();
+    }
+
+    /* GC_NEXT is a convenience macro for accessing the first word of an object.
+       Take the first list item, advance the list to the next item, and clear the next pointer. */
+    void * p = *valueAllocCache;
+    *valueAllocCache = GC_NEXT(p);
+    GC_NEXT(p) = nullptr;
+#else
+    void * p = allocBytes(sizeof(Value));
+#endif
+
+    nrValues++;
+    return (Value *) p;
+}
+
+
+[[gnu::always_inline]]
+Env & EvalState::allocEnv(size_t size)
+{
+    nrEnvs++;
+    nrValuesInEnvs += size;
+
+    Env * env;
+
+#if HAVE_BOEHMGC
+    if (size == 1) {
+        /* see allocValue for explanations. */
+        if (!*env1AllocCache) {
+            *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *));
+            if (!*env1AllocCache) throw std::bad_alloc();
+        }
+
+        void * p = *env1AllocCache;
+        *env1AllocCache = GC_NEXT(p);
+        GC_NEXT(p) = nullptr;
+        env = (Env *) p;
+    } else
+#endif
+        env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
+
+    env->type = Env::Plain;
+
+    /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
+
+    return *env;
+}
+
+
+[[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const Pos & pos)
 {
     forceValue(v, [&]() { return pos; });
@@ -66,6 +141,7 @@ void EvalState::forceValue(Value & v, Callable getPos)
 }
 
 
+[[gnu::always_inline]]
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceAttrs(v, [&]() { return pos; });
@@ -73,6 +149,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 
 
 template <typename Callable>
+[[gnu::always_inline]]
 inline void EvalState::forceAttrs(Value & v, Callable getPos)
 {
     forceValue(v, getPos);
@@ -81,6 +158,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos)
 }
 
 
+[[gnu::always_inline]]
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
@@ -88,18 +166,5 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError(pos, "value is %1% while a list was expected", v, *this);
 }
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-inline void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_MALLOC(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
 
 }

src/libexpr/eval.cc

@@ -63,9 +63,15 @@ static char * dupString(const char * s)
 }
 
 
-static char * dupStringWithLen(const char * s, size_t size)
+// When there's no need to write to the string, we can optimize away empty
+// string allocations.
+// This function handles makeImmutableStringWithLen(null, 0) by returning the
+// empty string.
+static const char * makeImmutableStringWithLen(const char * s, size_t size)
 {
     char * t;
+    if (size == 0)
+        return "";
 #if HAVE_BOEHMGC
     t = GC_STRNDUP(s, size);
 #else
@@ -75,6 +81,10 @@ static char * dupStringWithLen(const char * s, size_t size)
     return t;
 }
 
+static inline const char * makeImmutableString(std::string_view s) {
+    return makeImmutableStringWithLen(s.data(), s.size());
+}
+
 
 RootValue allocRootValue(Value * v)
 {
@@ -86,25 +96,20 @@ RootValue allocRootValue(Value * v)
 }
 
 
-void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
+void Value::print(std::ostream & str, std::set<const void *> * seen) const
 {
     checkInterrupt();
 
-    if (!active.insert(&v).second) {
+    switch (internalType) {
-        str << "<CYCLE>";
-        return;
-    }
-
-    switch (v.internalType) {
     case tInt:
-        str << v.integer;
+        str << integer;
         break;
     case tBool:
-        str << (v.boolean ? "true" : "false");
+        str << (boolean ? "true" : "false");
         break;
     case tString:
         str << "\"";
-        for (const char * i = v.string.s; *i; i++)
+        for (const char * i = string.s; *i; i++)
             if (*i == '\"' || *i == '\\') str << "\\" << *i;
             else if (*i == '\n') str << "\\n";
             else if (*i == '\r') str << "\\r";
@@ -114,30 +119,38 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         str << "\"";
         break;
     case tPath:
-        str << v.path; // !!! escaping?
+        str << path; // !!! escaping?
         break;
     case tNull:
         str << "null";
         break;
     case tAttrs: {
+        if (seen && !attrs->empty() && !seen->insert(attrs).second)
+            str << "«repeated»";
+        else {
             str << "{ ";
-        for (auto & i : v.attrs->lexicographicOrder()) {
+            for (auto & i : attrs->lexicographicOrder()) {
                 str << i->name << " = ";
-            printValue(str, active, *i->value);
+                i->value->print(str, seen);
                 str << "; ";
             }
             str << "}";
+        }
         break;
     }
     case tList1:
     case tList2:
     case tListN:
+        if (seen && listSize() && !seen->insert(listElems()).second)
+            str << "«repeated»";
+        else {
             str << "[ ";
-        for (auto v2 : v.listItems()) {
+            for (auto v2 : listItems()) {
-            printValue(str, active, *v2);
+                v2->print(str, seen);
                 str << " ";
             }
             str << "]";
+        }
         break;
     case tThunk:
     case tApp:
@@ -153,28 +166,32 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         str << "<PRIMOP-APP>";
         break;
     case tExternal:
-        str << *v.external;
+        str << *external;
         break;
     case tFloat:
-        str << v.fpoint;
+        str << fpoint;
         break;
     default:
         abort();
     }
+}
 
-    active.erase(&v);
+
+void Value::print(std::ostream & str, bool showRepeated) const
+{
+    std::set<const void *> seen;
+    print(str, showRepeated ? nullptr : &seen);
 }
 
 
 std::ostream & operator << (std::ostream & str, const Value & v)
 {
-    std::set<const Value *> active;
+    v.print(str, false);
-    printValue(str, active, v);
     return str;
 }
 
 
-const Value *getPrimOp(const Value &v) {
+const Value * getPrimOp(const Value &v) {
     const Value * primOp = &v;
     while (primOp->isPrimOpApp()) {
         primOp = primOp->primOpApp.left;
@@ -183,7 +200,7 @@ const Value *getPrimOp(const Value &v) {
     return primOp;
 }
 
-string showType(ValueType type)
+std::string_view showType(ValueType type)
 {
     switch (type) {
         case nInt: return "an integer";
@@ -202,20 +219,20 @@ string showType(ValueType type)
 }
 
 
-string showType(const Value & v)
+std::string showType(const Value & v)
 {
     switch (v.internalType) {
         case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
-            return fmt("the built-in function '%s'", string(v.primOp->name));
+            return fmt("the built-in function '%s'", std::string(v.primOp->name));
         case tPrimOpApp:
-            return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
+            return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
         case tThunk: return "a thunk";
         case tApp: return "a function application";
         case tBlackhole: return "a black hole";
     default:
-        return showType(v.type());
+        return std::string(showType(v.type()));
     }
 }
 
@@ -356,7 +373,7 @@ void initGC()
 
 /* Very hacky way to parse $NIX_PATH, which is colon-separated, but
    can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
-static Strings parseNixPath(const string & s)
+static Strings parseNixPath(const std::string & s)
 {
     Strings res;
 
@@ -419,6 +436,7 @@ EvalState::EvalState(
     , sBuilder(symbols.create("builder"))
     , sArgs(symbols.create("args"))
     , sContentAddressed(symbols.create("__contentAddressed"))
+    , sImpure(symbols.create("__impure"))
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
@@ -440,8 +458,10 @@ EvalState::EvalState(
     , regexCache(makeRegexCache())
 #if HAVE_BOEHMGC
     , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+    , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
 #else
     , valueAllocCache(std::make_shared<void *>(nullptr))
+    , env1AllocCache(std::make_shared<void *>(nullptr))
 #endif
     , baseEnv(allocEnv(128))
     , staticBaseEnv(new StaticEnv(false, 0))
@@ -490,23 +510,6 @@ EvalState::~EvalState()
 }
 
 
-void EvalState::requireExperimentalFeatureOnEvaluation(
-    const ExperimentalFeature & feature,
-    const std::string_view fName,
-    const Pos & pos)
-{
-    if (!settings.isExperimentalFeatureEnabled(feature)) {
-        throw EvalError({
-            .msg = hintfmt(
-                "Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.",
-                feature,
-                fName
-            ),
-            .errPos = pos
-        });
-    }
-}
-
 void EvalState::allowPath(const Path & path)
 {
     if (allowedPaths)
@@ -519,6 +522,14 @@ void EvalState::allowPath(const StorePath & storePath)
         allowedPaths->insert(store->toRealPath(storePath));
 }
 
+void EvalState::allowAndSetStorePathString(const StorePath &storePath, Value & v)
+{
+    allowPath(storePath);
+
+    auto path = store->printStorePath(storePath);
+    v.mkString(path, PathSet({path}));
+}
+
 Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
@@ -608,7 +619,7 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context)
 }
 
 
-Value * EvalState::addConstant(const string & name, Value & v)
+Value * EvalState::addConstant(const std::string & name, Value & v)
 {
     Value * v2 = allocValue();
     *v2 = v;
@@ -617,19 +628,19 @@ Value * EvalState::addConstant(const string & name, Value & v)
 }
 
 
-void EvalState::addConstant(const string & name, Value * v)
+void EvalState::addConstant(const std::string & name, Value * v)
 {
     staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
     baseEnv.values[baseEnvDispl++] = v;
-    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name;
     baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
 }
 
 
-Value * EvalState::addPrimOp(const string & name,
+Value * EvalState::addPrimOp(const std::string & name,
     size_t arity, PrimOpFun primOp)
 {
-    auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name;
     Symbol sym = symbols.create(name2);
 
     /* Hack to make constants lazy: turn them into a application of
@@ -677,7 +688,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
 }
 
 
-Value & EvalState::getBuiltin(const string & name)
+Value & EvalState::getBuiltin(const std::string & name)
 {
     return *baseEnv.values[0]->attrs->find(symbols.create(name))->value;
 }
@@ -746,7 +757,7 @@ void printEnvBindings(const StaticEnv &se, const Env &env, int lvl)
         // for the top level, don't print the double underscore ones; they are in builtins.
         for (auto i = se.vars.begin(); i != se.vars.end(); ++i)
         {
-            if (((string)i->first).substr(0,2) != "__")
+            if (((std::string)i->first).substr(0,2) != "__")
                 std::cout << i->first << " ";
         }
         std::cout << ANSI_NORMAL;
@@ -810,7 +821,7 @@ valmap * mapStaticEnvBindings(const StaticEnv &se, const Env &env)
    evaluator.  So here are some helper functions for throwing
    exceptions. */
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, EvalState &evalState))
+LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2, EvalState &evalState))
 {
     auto error = EvalError(s, s2);
 
@@ -833,11 +844,12 @@ void EvalState::debug_throw(Error e) {
     throw e;
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, Env & env, Expr &expr))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const Suggestions & suggestions, const char * s, const std::string & s2, Env & env, Expr &expr))
 {
     auto error = EvalError({
-        .msg = hintfmt(s),
+        .msg = hintfmt(s, s2),
-        .errPos = pos
+        .errPos = pos,
+        .suggestions = suggestions,
     });
 
     if (debuggerHook)
@@ -846,7 +858,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, Env &
     throw error;
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, EvalState &evalState))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2, EvalState &evalState))
 {
     auto error = EvalError({
         .msg = hintfmt(s, s2),
@@ -861,7 +873,20 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
     throw error;
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, Env & env, Expr &expr))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, Env & env, Expr &expr))
+{
+    auto error = EvalError({
+        .msg = hintfmt(s),
+        .errPos = pos
+    });
+
+    if (debuggerHook)
+        debuggerHook(&error, env, expr);
+
+    throw error;
+}
+
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2, Env & env, Expr &expr))
 {
     auto error = EvalError({
         .msg = hintfmt(s, s2),
@@ -874,7 +899,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
     throw error;
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3, EvalState &evalState))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2, const std::string & s3, EvalState &evalState))
 {
     auto error = EvalError({
         .msg = hintfmt(s, s2, s3),
@@ -889,7 +914,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
     throw error;
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, EvalState &evalState))
+LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2, const std::string & s3, EvalState &evalState))
 {
     auto error = EvalError({
         .msg = hintfmt(s, s2, s3),
@@ -947,11 +972,13 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
     throw error;
 }
 
+// LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v));
+
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2, Env & env, Expr &expr))
 {
     auto error = TypeError({
         .msg = hintfmt(s, fun.showNamePos(), s2),
-        .errPos = pos
+        .errPos = pos,
     });
 
     if (debuggerHook)
@@ -960,7 +987,21 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
     throw error;
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1, Env & env, Expr &expr))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const Suggestions & suggestions, const char * s, const ExprLambda & fun, const Symbol & s2, Env & env, Expr &expr))
+{
+    auto error = TypeError({
+        .msg = hintfmt(s, fun.showNamePos(), s2),
+        .errPos = pos,
+        .suggestions = suggestions,
+    });
+
+    if (debuggerHook)
+        debuggerHook(&error, env, expr);
+
+    throw error;
+}
+
+LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const std::string & s1, Env & env, Expr &expr))
 {
     auto error = AssertionError({
         .msg = hintfmt(s, s1),
@@ -973,7 +1014,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
     throw error;
 }
 
-LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1, Env & env, const Expr &expr))
+LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const std::string & s1, Env & env, const Expr &expr))
 {
     auto error = UndefinedVarError({
         .msg = hintfmt(s, s1),
@@ -986,7 +1027,7 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
     throw error;
 }
 
-LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1, Env & env, Expr &expr))
+LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const std::string & s1, Env & env, Expr &expr))
 {
     auto error = MissingArgumentError({
         .msg = hintfmt(s, s1),
@@ -999,18 +1040,18 @@ LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char
     throw error;
 }
 
-LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorTrace(Error & e, const char * s, const std::string & s2))
 {
     e.addTrace(std::nullopt, s, s2);
 }
 
-LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
+LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const std::string & s2))
 {
     e.addTrace(pos, s, s2);
 }
 
 LocalNoInline(std::unique_ptr<DebugTraceStacker>
-  makeDebugTraceStacker(EvalState &state, Expr &expr, Env &env, std::optional<ErrPos> pos, const char * s, const string & s2))
+  makeDebugTraceStacker(EvalState &state, Expr &expr, Env &env, std::optional<ErrPos> pos, const char * s, const std::string & s2))
 {
   return std::unique_ptr<DebugTraceStacker>(
       new DebugTraceStacker(
@@ -1034,7 +1075,7 @@ DebugTraceStacker::DebugTraceStacker(EvalState &evalState, DebugTrace t)
 
 void Value::mkString(std::string_view s)
 {
-    mkString(dupStringWithLen(s.data(), s.size()));
+    mkString(makeImmutableString(s));
 }
 
 
@@ -1065,7 +1106,7 @@ void Value::mkStringMove(const char * s, const PathSet & context)
 
 void Value::mkPath(std::string_view s)
 {
-    mkPath(dupStringWithLen(s.data(), s.size()));
+    mkPath(makeImmutableString(s));
 }
 
 
@@ -1095,53 +1136,6 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     }
 }
 
-
-Value * EvalState::allocValue()
-{
-    /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
-       GC_malloc_many returns a linked list of objects of the given size, where the first word
-       of each object is also the pointer to the next object in the list. This also means that we
-       have to explicitly clear the first word of every object we take. */
-    if (!*valueAllocCache) {
-        *valueAllocCache = GC_malloc_many(sizeof(Value));
-        if (!*valueAllocCache) throw std::bad_alloc();
-    }
-
-    /* GC_NEXT is a convenience macro for accessing the first word of an object.
-       Take the first list item, advance the list to the next item, and clear the next pointer. */
-    void * p = *valueAllocCache;
-    GC_PTR_STORE_AND_DIRTY(&*valueAllocCache, GC_NEXT(p));
-    GC_NEXT(p) = nullptr;
-
-    nrValues++;
-    auto v = (Value *) p;
-    return v;
-}
-
-
-Env & EvalState::allocEnv(size_t size)
-{
-
-    nrEnvs++;
-    nrValuesInEnvs += size;
-    Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->type = Env::Plain;
-
-    /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
-
-    return *env;
-}
-
-Env & fakeEnv(size_t size)
-{
-    // making a fake Env so we'll have one to pass to exception ftns.
-    // a placeholder until we can pass real envs everywhere they're needed.
-    Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->type = Env::Plain;
-
-    return *env;
-}
-
 void EvalState::mkList(Value & v, size_t size)
 {
     v.mkList(size);
@@ -1480,7 +1474,7 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
+static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
 {
     std::ostringstream out;
     bool first = true;
@@ -1531,8 +1525,15 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 }
             } else {
                 state.forceAttrs(*vAttrs, pos);
-                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
-                    throwEvalError(pos, "attribute '%1%' missing", name, env, *this);
+                    std::set<std::string> allAttrNames;
+                    for (auto & attr : *vAttrs->attrs)
+                        allAttrNames.insert(attr.name);
+                    throwEvalError(
+                        pos,
+                        Suggestions::bestMatches(allAttrNames, name),
+                        "attribute '%1%' missing", name, env, *this);
+                }
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -1647,9 +1648,16 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     /* Nope, so show the first unexpected argument to the
                        user. */
                     for (auto & i : *args[0]->attrs)
-                        if (!lambda.formals->has(i.name))
+                        if (!lambda.formals->has(i.name)) {
-                            throwTypeError(pos, "%1% called with unexpected argument '%2%'",
+                            std::set<std::string> formalNames;
+                            for (auto & formal : lambda.formals->formals)
+                                formalNames.insert(formal.name);
+                            throwTypeError(
+                                pos,
+                                Suggestions::bestMatches(formalNames, i.name),
+                                "%1% called with unexpected argument '%2%'",
                                 lambda, i.name, *fun.lambda.env, lambda);
+                        }
                     abort(); // can't happen
                 }
             }
@@ -1664,7 +1672,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                         makeDebugTraceStacker(*this, *lambda.body, env2, lambda.pos,
                                            "while evaluating %s",
                                            (lambda.name.set()
-                                               ? "'" + (string) lambda.name + "'"
+                                               ? "'" + (std::string) lambda.name + "'"
                                                : "anonymous lambda"))
                         : nullptr;
 
@@ -1673,7 +1681,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 if (loggerSettings.showTrace.get()) {
                     addErrorTrace(e, lambda.pos, "while evaluating %s",
                         (lambda.name.set()
-                            ? "'" + (string) lambda.name + "'"
+                            ? "'" + (const std::string &) lambda.name + "'"
                             : "anonymous lambda"));
                     addErrorTrace(e, pos, "from call site%s", "");
                 }
@@ -2175,13 +2183,22 @@ std::string_view EvalState::forceString(Value & v, const Pos & pos)
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
    name>. */
-std::pair<string, string> decodeContext(std::string_view s)
+NixStringContextElem decodeContext(const Store & store, std::string_view s)
 {
     if (s.at(0) == '!') {
         size_t index = s.find("!", 1);
-        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+        return {
+            store.parseStorePath(s.substr(index + 1)),
+            std::string(s.substr(1, index - 1)),
+        };
     } else
-        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+        return {
+            store.parseStorePath(
+                s.at(0) == '/'
+                ? s
+                : s.substr(1)),
+            "",
+        };
 }
 
 
@@ -2193,13 +2210,13 @@ void copyContext(const Value & v, PathSet & context)
 }
 
 
-std::vector<std::pair<Path, std::string>> Value::getContext()
+NixStringContext Value::getContext(const Store & store)
 {
-    std::vector<std::pair<Path, std::string>> res;
+    NixStringContext res;
     assert(internalType == tString);
     if (string.context)
         for (const char * * p = string.context; *p; ++p)
-            res.push_back(decodeContext(*p));
+            res.push_back(decodeContext(store, *p));
     return res;
 }
 
@@ -2238,7 +2255,7 @@ bool EvalState::isDerivation(Value & v)
 }
 
 
-std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
+std::optional<std::string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
     PathSet & context, bool coerceMore, bool copyToStore)
 {
     auto i = v.attrs->find(sToString);
@@ -2293,7 +2310,7 @@ BackedStringView EvalState::coerceToString(const Pos & pos, Value & v, PathSet &
         if (v.type() == nNull) return "";
 
         if (v.isList()) {
-            string result;
+            std::string result;
             for (auto [n, v2] : enumerate(v.listItems())) {
                 result += *coerceToString(pos, *v2, context, coerceMore, copyToStore);
                 if (n < v.listSize() - 1
@@ -2309,7 +2326,7 @@ BackedStringView EvalState::coerceToString(const Pos & pos, Value & v, PathSet &
 }
 
 
-string EvalState::copyPathToStore(PathSet & context, const Path & path)
+std::string EvalState::copyPathToStore(PathSet & context, const Path & path)
 {
     if (nix::isDerivation(path))
         throwEvalError("file names are not allowed to end in '%1%'", drvExtension, *this);
@@ -2335,13 +2352,25 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
 
 Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
-    string path = coerceToString(pos, v, context, false, false).toOwned();
+    auto path = coerceToString(pos, v, context, false, false).toOwned();
     if (path == "" || path[0] != '/')
         throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path, *this);
     return path;
 }
 
 
+StorePath EvalState::coerceToStorePath(const Pos & pos, Value & v, PathSet & context)
+{
+    auto path = coerceToString(pos, v, context, false, false).toOwned();
+    if (auto storePath = store->maybeParseStorePath(path))
+        return *storePath;
+    throw EvalError({
+        .msg = hintfmt("path '%1%' is not in the Nix store", path),
+        .errPos = pos
+    });
+}
+
+
 bool EvalState::eqValues(Value & v1, Value & v2)
 {
     forceValue(v1, noPos);
@@ -2508,11 +2537,11 @@ void EvalState::printStats()
                 for (auto & i : functionCalls) {
                     auto obj = list.object();
                     if (i.first->name.set())
-                        obj.attr("name", (const string &) i.first->name);
+                        obj.attr("name", (const std::string &) i.first->name);
                     else
                         obj.attr("name", nullptr);
                     if (i.first->pos) {
-                        obj.attr("file", (const string &) i.first->pos.file);
+                        obj.attr("file", (const std::string &) i.first->pos.file);
                         obj.attr("line", i.first->pos.line);
                         obj.attr("column", i.first->pos.column);
                     }
@@ -2524,7 +2553,7 @@ void EvalState::printStats()
                 for (auto & i : attrSelects) {
                     auto obj = list.object();
                     if (i.first) {
-                        obj.attr("file", (const string &) i.first.file);
+                        obj.attr("file", (const std::string &) i.first.file);
                         obj.attr("line", i.first.line);
                         obj.attr("column", i.first.column);
                     }
@@ -2541,7 +2570,7 @@ void EvalState::printStats()
 }
 
 
-string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
+std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
     throw TypeError({
         .msg = hintfmt("cannot coerce %1% to a string", showType()),

src/libexpr/eval.hh

@@ -89,7 +89,7 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sContentAddressed,
+        sContentAddressed, sImpure,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
         sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
@@ -150,9 +150,14 @@ private:
     /* Cache used by prim_match(). */
     std::shared_ptr<RegexCache> regexCache;
 
+#if HAVE_BOEHMGC
     /* Allocation cache for GC'd Value objects. */
     std::shared_ptr<void *> valueAllocCache;
 
+    /* Allocation cache for size-1 Env objects. */
+    std::shared_ptr<void *> env1AllocCache;
+#endif
+
 public:
 
     EvalState(
@@ -161,13 +166,7 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    void requireExperimentalFeatureOnEvaluation(
+    void addToSearchPath(const std::string & s);
-        const ExperimentalFeature &,
-        const std::string_view fName,
-        const Pos & pos
-    );
-
-    void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
 
@@ -178,6 +177,9 @@ public:
        the real store path if `store` is a chroot store. */
     void allowPath(const StorePath & storePath);
 
+    /* Allow access to a store path and return it as a string. */
+    void allowAndSetStorePathString(const StorePath & storePath, Value & v);
+
     /* Check whether access to a path is allowed and throw an error if
        not. Otherwise return the canonicalised path. */
     Path checkSourcePath(const Path & path);
@@ -268,7 +270,7 @@ public:
        set with attribute `type = "derivation"'). */
     bool isDerivation(Value & v);
 
-    std::optional<string> tryAttrsToString(const Pos & pos, Value & v,
+    std::optional<std::string> tryAttrsToString(const Pos & pos, Value & v,
         PathSet & context, bool coerceMore = false, bool copyToStore = true);
 
     /* String coercion.  Converts strings, paths and derivations to a
@@ -279,13 +281,16 @@ public:
         bool coerceMore = false, bool copyToStore = true,
         bool canonicalizePath = true);
 
-    string copyPathToStore(PathSet & context, const Path & path);
+    std::string copyPathToStore(PathSet & context, const Path & path);
 
     /* Path coercion.  Converts strings, paths and derivations to a
        path.  The result is guaranteed to be a canonicalised, absolute
        path.  Nothing is copied to the store. */
     Path coerceToPath(const Pos & pos, Value & v, PathSet & context);
 
+    /* Like coerceToPath, but the result must be a store path. */
+    StorePath coerceToStorePath(const Pos & pos, Value & v, PathSet & context);
+
 public:
 
     /* The base environment, containing the builtin functions and
@@ -301,18 +306,18 @@ private:
 
     void createBaseEnv();
 
-    Value * addConstant(const string & name, Value & v);
+    Value * addConstant(const std::string & name, Value & v);
 
-    void addConstant(const string & name, Value * v);
+    void addConstant(const std::string & name, Value * v);
 
-    Value * addPrimOp(const string & name,
+    Value * addPrimOp(const std::string & name,
         size_t arity, PrimOpFun primOp);
 
     Value * addPrimOp(PrimOp && primOp);
 
 public:
 
-    Value & getBuiltin(const string & name);
+    Value & getBuiltin(const std::string & name);
 
     struct Doc
     {
@@ -358,8 +363,8 @@ public:
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
 
     /* Allocation primitives. */
-    Value * allocValue();
+    inline Value * allocValue();
-    Env & allocEnv(size_t size);
+    inline Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
     Value * allocAttr(Value & vAttrs, std::string_view name);
@@ -442,12 +447,12 @@ class DebugTraceStacker {
 };
 
 /* Return a string representing the type of the value `v'. */
-string showType(ValueType type);
+std::string_view showType(ValueType type);
-string showType(const Value & v);
+std::string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
    name>. */
-std::pair<string, string> decodeContext(std::string_view s);
+NixStringContextElem decodeContext(const Store & store, std::string_view s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
@@ -531,3 +536,5 @@ extern EvalSettings evalSettings;
 static const std::string corepkgsPrefix{"/__corepkgs__/"};
 
 }
+
+#include "eval-inline.hh"

src/libexpr/flake/config.cc

@@ -1,5 +1,6 @@
 #include "flake.hh"
 #include "globals.hh"
+#include "fetch-settings.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -53,7 +54,7 @@ void ConfigFile::apply()
             auto trustedList = readTrustedList();
 
             bool trusted = false;
-            if (nix::settings.acceptFlakeConfig){
+            if (nix::fetchSettings.acceptFlakeConfig){
                 trusted = true;
             } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
                 trusted = *saved;

src/libexpr/flake/flake.cc

@@ -6,6 +6,7 @@
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "finally.hh"
+#include "fetch-settings.hh"
 
 namespace nix {
 
@@ -254,7 +255,7 @@ static Flake getFlake(
         for (auto & setting : *nixConfig->value->attrs) {
             forceTrivialValue(state, *setting.value, *setting.pos);
             if (setting.value->type() == nString)
-                flake.config.settings.insert({setting.name, string(state.forceStringNoCtx(*setting.value, *setting.pos))});
+                flake.config.settings.insert({setting.name, std::string(state.forceStringNoCtx(*setting.value, *setting.pos))});
             else if (setting.value->type() == nPath) {
                 PathSet emptyContext = {};
                 flake.config.settings.emplace(
@@ -315,7 +316,7 @@ LockedFlake lockFlake(
 
     FlakeCache flakeCache;
 
-    auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
+    auto useRegistries = lockFlags.useRegistries.value_or(fetchSettings.useRegistries);
 
     auto flake = getFlake(state, topRef, useRegistries, flakeCache);
 
@@ -501,7 +502,7 @@ LockedFlake lockFlake(
                            this input. */
                         debug("creating new input '%s'", inputPathS);
 
-                        if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
+                        if (!lockFlags.allowMutable && !input.ref->input.isLocked())
                             throw Error("cannot update flake input '%s' in pure mode", inputPathS);
 
                         if (input.isFlake) {
@@ -591,7 +592,7 @@ LockedFlake lockFlake(
             if (lockFlags.writeLockFile) {
                 if (auto sourcePath = topRef.input.getSourcePath()) {
                     if (!newLockFile.isImmutable()) {
-                        if (settings.warnDirty)
+                        if (fetchSettings.warnDirty)
                             warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
                     } else {
                         if (!lockFlags.updateLockFile)
@@ -618,7 +619,7 @@ LockedFlake lockFlake(
                         if (lockFlags.commitLockFile) {
                             std::string cm;
 
-                            cm = settings.commitLockFileSummary.get();
+                            cm = fetchSettings.commitLockFileSummary.get();
 
                             if (cm == "") {
                                 cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
@@ -650,7 +651,7 @@ LockedFlake lockFlake(
                            now. Corner case: we could have reverted from a
                            dirty to a clean tree! */
                         if (flake.lockedRef.input == prevLockedRef.input
-                            && !flake.lockedRef.input.isImmutable())
+                            && !flake.lockedRef.input.isLocked())
                             throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
                     }
                 } else
@@ -705,24 +706,45 @@ void callFlake(EvalState & state,
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
+    std::string flakeRefS(state.forceStringNoCtx(*args[0], pos));
-
-    string flakeRefS(state.forceStringNoCtx(*args[0], pos));
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
-    if (evalSettings.pureEval && !flakeRef.input.isImmutable())
+    if (evalSettings.pureEval && !flakeRef.input.isLocked())
-        throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
+        throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
 
     callFlake(state,
         lockFlake(state, flakeRef,
             LockFlags {
                 .updateLockFile = false,
-                .useRegistries = !evalSettings.pureEval && settings.useRegistries,
+                .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries,
                 .allowMutable  = !evalSettings.pureEval,
             }),
         v);
 }
 
-static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
+static RegisterPrimOp r2({
+    .name =  "__getFlake",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
+
+      ```nix
+      (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
+      ```
+
+      Unless impure evaluation is allowed (`--impure`), the flake reference
+      must be "locked", e.g. contain a Git revision or content hash. An
+      example of an unlocked usage is:
+
+      ```nix
+      (builtins.getFlake "github:edolstra/dwarffs").rev
+      ```
+
+      This function is only available if you enable the experimental feature
+      `flakes`.
+    )",
+    .fun = prim_getFlake,
+    .experimentalFeature = Xp::Flakes,
+});
 
 }
 

src/libexpr/flake/flakeref.cc

@@ -98,7 +98,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
     if (std::regex_match(url, match, flakeRegex)) {
         auto parsedURL = ParsedURL{
             .url = url,
-            .base = "flake:" + std::string(match[1]),
+            .base = "flake:" + match.str(1),
             .scheme = "flake",
             .authority = "",
             .path = match[1],
@@ -106,12 +106,12 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 
         return std::make_pair(
             FlakeRef(Input::fromURL(parsedURL), ""),
-            percentDecode(std::string(match[6])));
+            percentDecode(match.str(6)));
     }
 
     else if (std::regex_match(url, match, pathUrlRegex)) {
         std::string path = match[1];
-        std::string fragment = percentDecode(std::string(match[3]));
+        std::string fragment = percentDecode(match.str(3));
 
         if (baseDir) {
             /* Check if 'url' is a path (either absolute or relative

src/libexpr/flake/lockfile.cc

@@ -35,7 +35,7 @@ LockedNode::LockedNode(const nlohmann::json & json)
     , originalRef(getFlakeRef(json, "original", nullptr))
     , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
 {
-    if (!lockedRef.input.isImmutable())
+    if (!lockedRef.input.isLocked())
         throw Error("lockfile contains mutable lock '%s'",
             fetchers::attrsToJSON(lockedRef.input.toAttrs()));
 }
@@ -220,7 +220,7 @@ bool LockFile::isImmutable() const
     for (auto & i : nodes) {
         if (i == root) continue;
         auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
-        if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
+        if (lockedNode && !lockedNode->lockedRef.input.isLocked()) return false;
     }
 
     return true;

src/libexpr/get-drvs.cc

@@ -1,6 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
+#include "derivations.hh"
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 
@@ -11,8 +12,8 @@
 namespace nix {
 
 
-DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
+DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs)
-    : state(&state), attrs(attrs), attrPath(attrPath)
+    : state(&state), attrs(attrs), attrPath(std::move(attrPath))
 {
 }
 
@@ -22,7 +23,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 {
     auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
 
-    this->drvPath = store->printStorePath(drvPath);
+    this->drvPath = drvPath;
 
     auto drv = store->derivationFromPath(drvPath);
 
@@ -41,13 +42,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
         throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
     auto & [outputName, output] = *i;
 
-    auto optStorePath = output.path(*store, drv.name, outputName);
+    outPath = {output.path(*store, drv.name, outputName)};
-    if (optStorePath)
-        outPath = store->printStorePath(*optStorePath);
 }
 
 
-string DrvInfo::queryName() const
+std::string DrvInfo::queryName() const
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
@@ -58,7 +57,7 @@ string DrvInfo::queryName() const
 }
 
 
-string DrvInfo::querySystem() const
+std::string DrvInfo::querySystem() const
 {
     if (system == "" && attrs) {
         auto i = attrs->find(state->sSystem);
@@ -68,24 +67,35 @@ string DrvInfo::querySystem() const
 }
 
 
-string DrvInfo::queryDrvPath() const
+std::optional<StorePath> DrvInfo::queryDrvPath() const
 {
-    if (drvPath == "" && attrs) {
+    if (!drvPath && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
         PathSet context;
-        drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
+        if (i == attrs->end())
+            drvPath = {std::nullopt};
+        else
+            drvPath = {state->coerceToStorePath(*i->pos, *i->value, context)};
     }
-    return drvPath;
+    return drvPath.value_or(std::nullopt);
 }
 
 
-string DrvInfo::queryOutPath() const
+StorePath DrvInfo::requireDrvPath() const
+{
+    if (auto drvPath = queryDrvPath())
+        return *drvPath;
+    throw Error("derivation does not contain a 'drvPath' attribute");
+}
+
+
+StorePath DrvInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
         PathSet context;
         if (i != attrs->end())
-            outPath = state->coerceToPath(*i->pos, *i->value, context);
+            outPath = state->coerceToStorePath(*i->pos, *i->value, context);
     }
     if (!outPath)
         throw UnimplementedError("CA derivations are not yet supported");
@@ -93,7 +103,7 @@ string DrvInfo::queryOutPath() const
 }
 
 
-DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
+DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
@@ -103,9 +113,11 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 
             /* For each output... */
             for (auto elem : i->value->listItems()) {
+                std::string output(state->forceStringNoCtx(*elem, *i->pos));
+
+                if (withPaths) {
                     /* Evaluate the corresponding set. */
-                string name(state->forceStringNoCtx(*elem, *i->pos));
+                    Bindings::iterator out = attrs->find(state->symbols.create(output));
-                Bindings::iterator out = attrs->find(state->symbols.create(name));
                     if (out == attrs->end()) continue; // FIXME: throw error?
                     state->forceAttrs(*out->value, *i->pos);
 
@@ -113,10 +125,12 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
                     Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
                     if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
                     PathSet context;
-                outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
+                    outputs.emplace(output, state->coerceToStorePath(*outPath->pos, *outPath->value, context));
+                } else
+                    outputs.emplace(output, std::nullopt);
             }
         } else
-            outputs["out"] = queryOutPath();
+            outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt);
     }
     if (!onlyOutputsToInstall || !attrs)
         return outputs;
@@ -138,7 +152,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 }
 
 
-string DrvInfo::queryOutputName() const
+std::string DrvInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutputName);
@@ -190,7 +204,7 @@ bool DrvInfo::checkMeta(Value & v)
 }
 
 
-Value * DrvInfo::queryMeta(const string & name)
+Value * DrvInfo::queryMeta(const std::string & name)
 {
     if (!getMeta()) return 0;
     Bindings::iterator a = meta->find(state->symbols.create(name));
@@ -199,7 +213,7 @@ Value * DrvInfo::queryMeta(const string & name)
 }
 
 
-string DrvInfo::queryMetaString(const string & name)
+std::string DrvInfo::queryMetaString(const std::string & name)
 {
     Value * v = queryMeta(name);
     if (!v || v->type() != nString) return "";
@@ -207,7 +221,7 @@ string DrvInfo::queryMetaString(const string & name)
 }
 
 
-NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
+NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -221,7 +235,7 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
     return def;
 }
 
-NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
+NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -236,7 +250,7 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 }
 
 
-bool DrvInfo::queryMetaBool(const string & name, bool def)
+bool DrvInfo::queryMetaBool(const std::string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -251,7 +265,7 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 }
 
 
-void DrvInfo::setMeta(const string & name, Value * v)
+void DrvInfo::setMeta(const std::string & name, Value * v)
 {
     getMeta();
     auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0));
@@ -266,7 +280,7 @@ void DrvInfo::setMeta(const string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
-typedef set<Bindings *> Done;
+typedef std::set<Bindings *> Done;
 
 
 /* Evaluate value `v'.  If it evaluates to a set of type `derivation',
@@ -274,7 +288,7 @@ typedef set<Bindings *> Done;
    The result boolean indicates whether it makes sense
    for the caller to recursively search for derivations in `v'. */
 static bool getDerivation(EvalState & state, Value & v,
-    const string & attrPath, DrvInfos & drvs, Done & done,
+    const std::string & attrPath, DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     try {
@@ -311,7 +325,7 @@ std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
 }
 
 
-static string addToPath(const string & s1, const string & s2)
+static std::string addToPath(const std::string & s1, const std::string & s2)
 {
     return s1.empty() ? s2 : s1 + "." + s2;
 }
@@ -321,7 +335,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
 
 
 static void getDerivations(EvalState & state, Value & vIn,
-    const string & pathPrefix, Bindings & autoArgs,
+    const std::string & pathPrefix, Bindings & autoArgs,
     DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
@@ -346,7 +360,7 @@ static void getDerivations(EvalState & state, Value & vIn,
             debug("evaluating attribute '%1%'", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
-            string pathPrefix2 = addToPath(pathPrefix, i->name);
+            std::string pathPrefix2 = addToPath(pathPrefix, i->name);
             if (combineChannels)
                 getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
             else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
@@ -364,7 +378,7 @@ static void getDerivations(EvalState & state, Value & vIn,
 
     else if (v.type() == nList) {
         for (auto [n, elem] : enumerate(v.listItems())) {
-            string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
+            std::string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
             if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures))
                 getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
@@ -374,7 +388,7 @@ static void getDerivations(EvalState & state, Value & vIn,
 }
 
 
-void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
     Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures)
 {
     Done done;

src/libexpr/get-drvs.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "eval.hh"
+#include "path.hh"
 
 #include <string>
 #include <map>
@@ -12,16 +13,16 @@ namespace nix {
 struct DrvInfo
 {
 public:
-    typedef std::map<string, Path> Outputs;
+    typedef std::map<std::string, std::optional<StorePath>> Outputs;
 
 private:
     EvalState * state;
 
-    mutable string name;
+    mutable std::string name;
-    mutable string system;
+    mutable std::string system;
-    mutable string drvPath;
+    mutable std::optional<std::optional<StorePath>> drvPath;
-    mutable std::optional<string> outPath;
+    mutable std::optional<StorePath> outPath;
-    mutable string outputName;
+    mutable std::string outputName;
     Outputs outputs;
 
     bool failed = false; // set if we get an AssertionError
@@ -33,36 +34,38 @@ private:
     bool checkMeta(Value & v);
 
 public:
-    string attrPath; /* path towards the derivation */
+    std::string attrPath; /* path towards the derivation */
 
     DrvInfo(EvalState & state) : state(&state) { };
-    DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs);
+    DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
     DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
-    string queryName() const;
+    std::string queryName() const;
-    string querySystem() const;
+    std::string querySystem() const;
-    string queryDrvPath() const;
+    std::optional<StorePath> queryDrvPath() const;
-    string queryOutPath() const;
+    StorePath requireDrvPath() const;
-    string queryOutputName() const;
+    StorePath queryOutPath() const;
-    /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
+    std::string queryOutputName() const;
-    Outputs queryOutputs(bool onlyOutputsToInstall = false);
+    /** Return the unordered map of output names to (optional) output paths.
+     * The "outputs to install" are determined by `meta.outputsToInstall`. */
+    Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();
-    Value * queryMeta(const string & name);
+    Value * queryMeta(const std::string & name);
-    string queryMetaString(const string & name);
+    std::string queryMetaString(const std::string & name);
-    NixInt queryMetaInt(const string & name, NixInt def);
+    NixInt queryMetaInt(const std::string & name, NixInt def);
-    NixFloat queryMetaFloat(const string & name, NixFloat def);
+    NixFloat queryMetaFloat(const std::string & name, NixFloat def);
-    bool queryMetaBool(const string & name, bool def);
+    bool queryMetaBool(const std::string & name, bool def);
-    void setMeta(const string & name, Value * v);
+    void setMeta(const std::string & name, Value * v);
 
     /*
     MetaInfo queryMetaInfo(EvalState & state) const;
     MetaValue queryMetaInfo(EvalState & state, const string & name) const;
     */
 
-    void setName(const string & s) { name = s; }
+    void setName(const std::string & s) { name = s; }
-    void setDrvPath(const string & s) { drvPath = s; }
+    void setDrvPath(StorePath path) { drvPath = {{std::move(path)}}; }
-    void setOutPath(const string & s) { outPath = s; }
+    void setOutPath(StorePath path) { outPath = {{std::move(path)}}; }
 
     void setFailed() { failed = true; };
     bool hasFailed() { return failed; };
@@ -70,9 +73,9 @@ public:
 
 
 #if HAVE_BOEHMGC
-typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
+typedef std::list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
 #else
-typedef list<DrvInfo> DrvInfos;
+typedef std::list<DrvInfo> DrvInfos;
 #endif
 
 
@@ -81,7 +84,7 @@ typedef list<DrvInfo> DrvInfos;
 std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
-void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
     Bindings & autoArgs, DrvInfos & drvs,
     bool ignoreAssertionFailures);
 

src/libexpr/lexer.l

@@ -28,6 +28,13 @@ using namespace nix;
 
 namespace nix {
 
+static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
+{
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
+}
+
+#define CUR_POS makeCurPos(*yylloc, data)
+
 // backup to recover from yyless(0)
 YYLTYPE prev_yylloc;
 
@@ -37,7 +44,6 @@ static void initLoc(YYLTYPE * loc)
     loc->first_column = loc->last_column = 1;
 }
 
-
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
     prev_yylloc = *loc;
@@ -147,14 +153,20 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError("invalid integer '%1%'", yytext);
+                  throw ParseError({
+                      .msg = hintfmt("invalid integer '%1%'", yytext),
+                      .errPos = CUR_POS,
+                  });
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError("invalid float '%1%'", yytext);
+                  throw ParseError({
+                      .msg = hintfmt("invalid float '%1%'", yytext),
+                      .errPos = CUR_POS,
+                  });
               return FLOAT;
             }
 
@@ -280,7 +292,10 @@ or          { return OR_KW; }
 
 <INPATH_SLASH>{ANY} |
 <INPATH_SLASH><<EOF>> {
-  throw ParseError("path has a trailing slash");
+  throw ParseError({
+      .msg = hintfmt("path has a trailing slash"),
+      .errPos = CUR_POS,
+  });
 }
 
 {SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }

src/libexpr/nixexpr.cc

@@ -18,10 +18,10 @@ std::ostream & operator << (std::ostream & str, const Expr & e)
     return str;
 }
 
-static void showString(std::ostream & str, const string & s)
+static void showString(std::ostream & str, std::string_view s)
 {
     str << '"';
-    for (auto c : (string) s)
+    for (auto c : s)
         if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
         else if (c == '\n') str << "\\n";
         else if (c == '\r') str << "\\r";
@@ -30,7 +30,7 @@ static void showString(std::ostream & str, const string & s)
     str << '"';
 }
 
-static void showId(std::ostream & str, const string & s)
+static void showId(std::ostream & str, std::string_view s)
 {
     if (s.empty())
         str << "\"\"";
@@ -105,11 +105,18 @@ void ExprAttrs::show(std::ostream & str) const
 {
     if (recursive) str << "rec ";
     str << "{ ";
-    for (auto & i : attrs)
+    typedef const decltype(attrs)::value_type * Attr;
-        if (i.second.inherited)
+    std::vector<Attr> sorted;
-            str << "inherit " << i.first << " " << "; ";
+    for (auto & i : attrs) sorted.push_back(&i);
+        std::sort(sorted.begin(), sorted.end(), [](Attr a, Attr b) {
+            return (const std::string &) a->first < (const std::string &) b->first;
+        });
+    for (auto & i : sorted) {
+        if (i->second.inherited)
+            str << "inherit " << i->first << " " << "; ";
         else
-            str << i.first << " = " << *i.second.e << "; ";
+            str << i->first << " = " << *i->second.e << "; ";
+    }
     for (auto & i : dynamicAttrs)
         str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; ";
     str << "}";
@@ -213,7 +220,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
         auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
         switch (pos.origin) {
             case foFile:
-                f % (string) pos.file;
+                f % (const std::string &) pos.file;
                 break;
             case foStdin:
             case foString:
@@ -229,7 +236,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
 }
 
 
-string showAttrPath(const AttrPath & attrPath)
+std::string showAttrPath(const AttrPath & attrPath)
 {
     std::ostringstream out;
     bool first = true;
@@ -518,9 +525,9 @@ void ExprLambda::setName(Symbol & name)
 }
 
 
-string ExprLambda::showNamePos() const
+std::string ExprLambda::showNamePos() const
 {
-    return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
+    return fmt("%1% at %2%", name.set() ? "'" + (std::string) name + "'" : "anonymous function", pos);
 }
 
 

src/libexpr/nixexpr.hh

@@ -24,21 +24,23 @@ extern std::function<void(const Error * error, const Env & env, const Expr & exp
 
 struct Pos
 {
-    FileOrigin origin;
     Symbol file;
-    unsigned int line, column;
+    uint32_t line;
-    Pos() : origin(foString), line(0), column(0) { };
+    FileOrigin origin:2;
-    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
+    uint32_t column:30;
-        : origin(origin), file(file), line(line), column(column) { };
+    Pos() : line(0), origin(foString), column(0) { };
+    Pos(FileOrigin origin, const Symbol & file, uint32_t line, uint32_t column)
+        : file(file), line(line), origin(origin), column(column) { };
     operator bool() const
     {
         return line != 0;
     }
+
     bool operator < (const Pos & p2) const
     {
         if (!line) return p2.line;
         if (!p2.line) return false;
-        int d = ((string) file).compare((string) p2.file);
+        int d = ((const std::string &) file).compare((const std::string &) p2.file);
         if (d < 0) return true;
         if (d > 0) return false;
         if (line < p2.line) return true;
@@ -69,7 +71,7 @@ struct AttrName
 
 typedef std::vector<AttrName> AttrPath;
 
-string showAttrPath(const AttrPath & attrPath);
+std::string showAttrPath(const AttrPath & attrPath);
 
 
 /* Abstract syntax of Nix expressions. */
@@ -116,7 +118,7 @@ struct ExprFloat : Expr
 
 struct ExprString : Expr
 {
-    string s;
+    std::string s;
     Value v;
     ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); };
     Value * maybeThunk(EvalState & state, Env & env);
@@ -126,9 +128,9 @@ struct ExprString : Expr
 
 struct ExprPath : Expr
 {
-    string s;
+    std::string s;
     Value v;
-    ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); };
+    ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); };
     Value * maybeThunk(EvalState & state, Env & env);
     const Pos* getPos() const { return 0; }
     COMMON_METHODS
@@ -262,7 +264,7 @@ struct ExprLambda : Expr
     {
     };
     void setName(Symbol & name);
-    string showNamePos() const;
+    std::string showNamePos() const;
     inline bool hasFormals() const { return formals != nullptr; }
     const Pos* getPos() const { return &pos; }
     COMMON_METHODS
@@ -356,8 +358,8 @@ struct ExprConcatStrings : Expr
 {
     Pos pos;
     bool forceString;
-    vector<std::pair<Pos, Expr *> > * es;
+    std::vector<std::pair<Pos, Expr *> > * es;
-    ExprConcatStrings(const Pos & pos, bool forceString, vector<std::pair<Pos, Expr *> > * es)
+    ExprConcatStrings(const Pos & pos, bool forceString, std::vector<std::pair<Pos, Expr *> > * es)
         : pos(pos), forceString(forceString), es(es) { };
     const Pos* getPos() const { return &pos; }
     COMMON_METHODS
@@ -390,15 +392,19 @@ struct StaticEnv
 
     void sort()
     {
-        std::sort(vars.begin(), vars.end(),
+        std::stable_sort(vars.begin(), vars.end(),
             [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
     }
 
     void deduplicate()
     {
-        const auto last = std::unique(vars.begin(), vars.end(),
+        auto it = vars.begin(), jt = it, end = vars.end();
-            [] (const Vars::value_type & a, const Vars::value_type & b) { return a.first == b.first; });
+        while (jt != end) {
-        vars.erase(last, vars.end());
+            *it = *jt++;
+            while (jt != end && it->first == jt->first) *it = *jt++;
+            it++;
+        }
+        vars.erase(it, end);
     }
 
     Vars::const_iterator find(const Symbol & name) const

src/libexpr/parser.y

@@ -194,7 +194,7 @@ static Formals * toFormals(ParseData & data, ParserFormals * formals,
 
 
 static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols,
-    vector<std::pair<Pos, std::variant<Expr *, StringToken> > > & es)
+    std::vector<std::pair<Pos, std::variant<Expr *, StringToken> > > & es)
 {
     if (es.empty()) return new ExprString("");
 
@@ -234,7 +234,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols,
     }
 
     /* Strip spaces from each line. */
-    vector<std::pair<Pos, Expr *> > * es2 = new vector<std::pair<Pos, Expr *> >;
+    std::vector<std::pair<Pos, Expr *> > * es2 = new std::vector<std::pair<Pos, Expr *> >;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
@@ -245,7 +245,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols,
         es2->emplace_back(i->first, e);
     };
     const auto trimString = [&] (const StringToken & t) {
-        string s2;
+        std::string s2;
         for (size_t j = 0; j < t.l; ++j) {
             if (atStartOfLine) {
                 if (t.p[j] == ' ') {
@@ -269,9 +269,9 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols,
         /* Remove the last line if it is empty and consists only of
            spaces. */
         if (n == 1) {
-            string::size_type p = s2.find_last_of('\n');
+            std::string::size_type p = s2.find_last_of('\n');
-            if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos)
+            if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos)
-                s2 = string(s2, 0, p + 1);
+                s2 = std::string(s2, 0, p + 1);
         }
 
         es2->emplace_back(i->first, new ExprString(s2));
@@ -416,7 +416,7 @@ expr_op
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); }
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
+    { $$ = new ExprConcatStrings(CUR_POS, false, new std::vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
   | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
   | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
   | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
@@ -467,7 +467,7 @@ expr_simple
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
-      string path($1.p + 1, $1.l - 2);
+      std::string path($1.p + 1, $1.l - 2);
       $$ = new ExprCall(CUR_POS,
           new ExprVar(data->symbols.create("__findFile")),
           {new ExprVar(data->symbols.create("__nixPath")),
@@ -480,7 +480,7 @@ expr_simple
               .msg = hintfmt("URL literals are disabled"),
               .errPos = CUR_POS
           });
-      $$ = new ExprString(string($1));
+      $$ = new ExprString(std::string($1));
   }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
@@ -495,19 +495,19 @@ expr_simple
   ;
 
 string_parts
-  : STR { $$ = new ExprString(string($1)); }
+  : STR { $$ = new ExprString(std::string($1)); }
   | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
   | { $$ = new ExprString(""); }
   ;
 
 string_parts_interpolated
   : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(string($2))); }
+  { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); }
   | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); }
+  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); }
   | STR DOLLAR_CURLY expr '}' {
-      $$ = new vector<std::pair<Pos, Expr *> >;
+      $$ = new std::vector<std::pair<Pos, Expr *> >;
-      $$->emplace_back(makeCurPos(@1, data), new ExprString(string($1)));
+      $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1)));
       $$->emplace_back(makeCurPos(@2, data), $3);
     }
   ;
@@ -521,7 +521,7 @@ path_start
     $$ = new ExprPath(path);
   }
   | HPATH {
-    Path path(getHome() + string($1.p + 1, $1.l - 1));
+    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
     $$ = new ExprPath(path);
   }
   ;
@@ -529,7 +529,7 @@ path_start
 ind_string_parts
   : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
   | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
-  | { $$ = new vector<std::pair<Pos, std::variant<Expr *, StringToken> > >; }
+  | { $$ = new std::vector<std::pair<Pos, std::variant<Expr *, StringToken> > >; }
   ;
 
 binds
@@ -583,9 +583,9 @@ attrpath
       } else
           $$->push_back(AttrName($3));
     }
-  | attr { $$ = new vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); }
+  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); }
   | string_attr
-    { $$ = new vector<AttrName>;
+    { $$ = new std::vector<AttrName>;
       ExprString *str = dynamic_cast<ExprString *>($1);
       if (str) {
           $$->push_back(AttrName(data->symbols.create(str->s)));
@@ -739,16 +739,16 @@ Expr * EvalState::parseStdin()
 }
 
 
-void EvalState::addToSearchPath(const string & s)
+void EvalState::addToSearchPath(const std::string & s)
 {
     size_t pos = s.find('=');
-    string prefix;
+    std::string prefix;
     Path path;
-    if (pos == string::npos) {
+    if (pos == std::string::npos) {
         path = s;
     } else {
-        prefix = string(s, 0, pos);
+        prefix = std::string(s, 0, pos);
-        path = string(s, pos + 1);
+        path = std::string(s, pos + 1);
     }
 
     searchPath.emplace_back(prefix, path);

src/libexpr/primops.cc

@@ -43,8 +43,8 @@ StringMap EvalState::realiseContext(const PathSet & context)
     StringMap res;
 
     for (auto & i : context) {
-        auto [ctxS, outputName] = decodeContext(i);
+        auto [ctx, outputName] = decodeContext(*store, i);
-        auto ctx = store->parseStorePath(ctxS);
+        auto ctxS = store->printStorePath(ctx);
         if (!store->isValidPath(ctx))
             debug_throw(InvalidPathError(store->printStorePath(ctx)));
         if (!outputName.empty() && ctx.isDerivation()) {
@@ -141,7 +141,7 @@ static void mkOutputString(
     BindingsBuilder & attrs,
     const StorePath & drvPath,
     const BasicDerivation & drv,
-    const std::pair<string, DerivationOutput> & o)
+    const std::pair<std::string, DerivationOutput> & o)
 {
     auto optOutputPath = o.second.path(*state.store, drv.name, o.first);
     attrs.alloc(o.first).mkString(
@@ -314,7 +314,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
 {
     auto path = realisePath(state, pos, *args[0]);
 
-    string sym(state.forceStringNoCtx(*args[1], pos));
+    std::string sym(state.forceStringNoCtx(*args[1], pos));
 
     void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
     if (!handle)
@@ -386,7 +386,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
 static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    string t;
+    std::string t;
     switch (args[0]->type()) {
         case nInt: t = "int"; break;
         case nBool: t = "bool"; break;
@@ -575,9 +575,9 @@ struct CompareValues
 
 
 #if HAVE_BOEHMGC
-typedef list<Value *, gc_allocator<Value *> > ValueList;
+typedef std::list<Value *, gc_allocator<Value *> > ValueList;
 #else
-typedef list<Value *> ValueList;
+typedef std::list<Value *> ValueList;
 #endif
 
 
@@ -655,7 +655,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     // `doneKeys' doesn't need to be a GC root, because its values are
     // reachable from res.
     auto cmp = CompareValues(state);
-    set<Value *, decltype(cmp)> doneKeys(cmp);
+    std::set<Value *, decltype(cmp)> doneKeys(cmp);
     while (!workSet.empty()) {
         Value * e = *(workSet.begin());
         workSet.pop_front();
@@ -695,7 +695,32 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
 
 static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
     .name = "__genericClosure",
+    .args = {"attrset"},
     .arity = 1,
+    .doc = R"(
+      Take an *attrset* with values named `startSet` and `operator` in order to
+      return a *list of attrsets* by starting with the `startSet`, recursively
+      applying the `operator` function to each element. The *attrsets* in the
+      `startSet` and produced by the `operator` must each contain value named
+      `key` which are comparable to each other. The result is produced by
+      repeatedly calling the operator for each element encountered with a
+      unique key, terminating when no new elements are produced. For example,
+
+      ```
+      builtins.genericClosure {
+        startSet = [ {key = 5;} ];
+        operator = item: [{
+          key = if (item.key / 2 ) * 2 == item.key
+               then item.key / 2
+               else 3 * item.key + 1;
+        }];
+      }
+      ```
+      evaluates to
+      ```
+      [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
+      ```
+      )",
     .fun = prim_genericClosure,
 });
 
@@ -742,7 +767,7 @@ static RegisterPrimOp primop_abort({
     .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v)
     {
         PathSet context;
-        string s = state.coerceToString(pos, *args[0], context).toOwned();
+        auto s = state.coerceToString(pos, *args[0], context).toOwned();
         state.debug_throw(Abort("evaluation aborted with the following error message: '%1%'", s));
     }
 });
@@ -760,7 +785,7 @@ static RegisterPrimOp primop_throw({
     .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v)
     {
       PathSet context;
-      string s = state.coerceToString(pos, *args[0], context).toOwned();
+      auto s = state.coerceToString(pos, *args[0], context).toOwned();
       state.debug_throw(ThrownError(s));
     }
 });
@@ -861,7 +886,7 @@ static RegisterPrimOp primop_tryEval({
 /* Return an environment variable.  Use with care. */
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    string name(state.forceStringNoCtx(*args[0], pos));
+    std::string name(state.forceStringNoCtx(*args[0], pos));
     v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or(""));
 }
 
@@ -970,7 +995,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         pos
     );
 
-    string drvName;
+    std::string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
@@ -999,16 +1024,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     PathSet context;
 
     bool contentAddressed = false;
+    bool isImpure = false;
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    auto ingestionMethod = FileIngestionMethod::Flat;
+    std::optional<FileIngestionMethod> ingestionMethod;
 
     StringSet outputs;
     outputs.insert("out");
 
     for (auto & i : args[0]->attrs->lexicographicOrder()) {
         if (i->name == state.sIgnoreNulls) continue;
-        const string & key = i->name;
+        const std::string & key = i->name;
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string_view s) {
@@ -1061,12 +1087,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                     settings.requireExperimentalFeature(Xp::CaDerivations);
             }
 
+            else if (i->name == state.sImpure) {
+                isImpure = state.forceBool(*i->value, pos);
+                if (isImpure)
+                    settings.requireExperimentalFeature(Xp::ImpureDerivations);
+            }
+
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
             else if (i->name == state.sArgs) {
                 state.forceList(*i->value, pos);
                 for (auto elem : i->value->listItems()) {
-                    string s = state.coerceToString(posDrvName, *elem, context, true).toOwned();
+                    auto s = state.coerceToString(posDrvName, *elem, context, true).toOwned();
                     drv.args.push_back(s);
                 }
             }
@@ -1153,8 +1185,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
-            std::pair<string, string> ctx = decodeContext(path);
+            auto ctx = decodeContext(*state.store, path);
-            drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second);
+            drv.inputDrvs[ctx.first].insert(ctx.second);
         }
 
         /* Otherwise it's a source file. */
@@ -1193,30 +1225,43 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 .errPos = posDrvName
             }));
 
-        std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
+        auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
-        Hash h = newHashAllowEmpty(*outputHash, ht);
 
-        auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
+        auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
+        auto outPath = state.store->makeFixedOutputPath(method, h, drvName);
         drv.env["out"] = state.store->printStorePath(outPath);
-        drv.outputs.insert_or_assign("out", DerivationOutput {
+        drv.outputs.insert_or_assign("out",
-                .output = DerivationOutputCAFixed {
+            DerivationOutput::CAFixed {
                 .hash = FixedOutputHash {
-                        .method = ingestionMethod,
+                    .method = method,
                     .hash = std::move(h),
                 },
-                },
             });
     }
 
-    else if (contentAddressed) {
+    else if (contentAddressed || isImpure) {
-        HashType ht = parseHashType(outputHashAlgo);
+        if (contentAddressed && isImpure)
+            throw EvalError({
+                .msg = hintfmt("derivation cannot be both content-addressed and impure"),
+                .errPos = posDrvName
+            });
+
+        auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256);
+        auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
+
         for (auto & i : outputs) {
             drv.env[i] = hashPlaceholder(i);
-            drv.outputs.insert_or_assign(i, DerivationOutput {
+            if (isImpure)
-                .output = DerivationOutputCAFloating {
+                drv.outputs.insert_or_assign(i,
-                    .method = ingestionMethod,
+                    DerivationOutput::Impure {
+                        .method = method,
+                        .hashType = ht,
+                    });
+            else
+                drv.outputs.insert_or_assign(i,
+                    DerivationOutput::CAFloating {
+                        .method = method,
                         .hashType = ht,
-                },
                     });
         }
     }
@@ -1231,44 +1276,29 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         for (auto & i : outputs) {
             drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
-                DerivationOutput {
+                DerivationOutput::Deferred { });
-                    .output = DerivationOutputInputAddressed {
-                        .path = StorePath::dummy,
-                    },
-                });
         }
 
-        // Regular, non-CA derivation should always return a single hash and not
-        // hash per output.
         auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
-        std::visit(overloaded {
+        switch (hashModulo.kind) {
-            [&](Hash & h) {
+        case DrvHash::Kind::Regular:
             for (auto & i : outputs) {
+                auto h = hashModulo.hashes.at(i);
                 auto outPath = state.store->makeOutputPath(i, h, drvName);
                 drv.env[i] = state.store->printStorePath(outPath);
-                    drv.outputs.insert_or_assign(i,
+                drv.outputs.insert_or_assign(
-                        DerivationOutput {
+                    i,
-                            .output = DerivationOutputInputAddressed {
+                    DerivationOutputInputAddressed {
                         .path = std::move(outPath),
-                            },
                     });
             }
-            },
+            break;
-            [&](CaOutputHashes &) {
+            ;
-                // Shouldn't happen as the toplevel derivation is not CA.
+        case DrvHash::Kind::Deferred:
-                assert(false);
-            },
-            [&](DeferredHash &) {
             for (auto & i : outputs) {
-                    drv.outputs.insert_or_assign(i,
+                drv.outputs.insert_or_assign(i, DerivationOutputDeferred {});
-                        DerivationOutput {
+            }
-                            .output = DerivationOutputDeferred{},
-                        });
         }
-            },
-        },
-        hashModulo);
-
     }
 
     /* Write the resulting term into the Nix store directory. */
@@ -1279,12 +1309,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store derivations, so we can't
-       read them later.
+       read them later. */
-
+    {
-       However, we don't bother doing this for floating CA derivations because
+        auto h = hashDerivationModulo(*state.store, drv, false);
-       their "hash modulo" is indeterminate until built. */
-    if (drv.type() != DerivationType::CAFloating) {
-        auto h = hashDerivationModulo(*state.store, Derivation(drv), false);
         drvHashes.lock()->insert_or_assign(drvPath, h);
     }
 
@@ -1475,8 +1502,8 @@ static RegisterPrimOp primop_dirOf({
 static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto path = realisePath(state, pos, *args[0]);
-    string s = readFile(path);
+    auto s = readFile(path);
-    if (s.find((char) 0) != string::npos)
+    if (s.find((char) 0) != std::string::npos)
         state.debug_throw(Error("the contents of the file '%1%' cannot be represented as a Nix string", path));
     StorePathSet refs;
     if (state.store->isInStore(path)) {
@@ -1509,7 +1536,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
     for (auto v2 : args[0]->listItems()) {
         state.forceAttrs(*v2, pos);
 
-        string prefix;
+        std::string prefix;
         Bindings::iterator i = v2->attrs->find(state.sPrefix);
         if (i != v2->attrs->end())
             prefix = state.forceStringNoCtx(*i->value, pos);
@@ -1523,7 +1550,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         );
 
         PathSet context;
-        string path = state.coerceToString(pos, *i->value, context, false, false).toOwned();
+        auto path = state.coerceToString(pos, *i->value, context, false, false).toOwned();
 
         try {
             auto rewrites = state.realiseContext(context);
@@ -1789,8 +1816,8 @@ static RegisterPrimOp primop_fromJSON({
 static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    string name(state.forceStringNoCtx(*args[0], pos));
+    std::string name(state.forceStringNoCtx(*args[0], pos));
-    string contents(state.forceString(*args[1], context, pos));
+    std::string contents(state.forceString(*args[1], context, pos));
 
     StorePathSet refs;
 
@@ -1898,7 +1925,7 @@ static RegisterPrimOp primop_toFile({
 static void addPath(
     EvalState & state,
     const Pos & pos,
-    const string & name,
+    const std::string & name,
     Path path,
     Value * filterFun,
     FileIngestionMethod method,
@@ -1954,20 +1981,15 @@ static void addPath(
         if (expectedHash)
             expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
 
-        Path dstPath;
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            dstPath = state.store->printStorePath(settings.readOnlyMode
+            StorePath dstPath = settings.readOnlyMode
                 ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
-                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs));
+                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs);
-            if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+            if (expectedHash && expectedStorePath != dstPath)
                 state.debug_throw(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
+            state.allowAndSetStorePathString(dstPath, v);
         } else
-            dstPath = state.store->printStorePath(*expectedStorePath);
+            state.allowAndSetStorePathString(*expectedStorePath, v);
-
-        v.mkString(dstPath, {dstPath});
-
-        state.allowPath(dstPath);
-
     } catch (Error & e) {
         e.addTrace(pos, "while adding path '%s'", path);
         throw;
@@ -2051,14 +2073,14 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceAttrs(*args[0], pos);
     Path path;
-    string name;
+    std::string name;
     Value * filterFun = nullptr;
     auto method = FileIngestionMethod::Recursive;
     std::optional<Hash> expectedHash;
     PathSet context;
 
     for (auto & attr : *args[0]->attrs) {
-        const string & n(attr.name);
+        auto & n(attr.name);
         if (n == "path")
             path = state.coerceToPath(*attr.pos, *attr.value, context);
         else if (attr.name == state.sName)
@@ -3651,7 +3673,7 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * *
     auto sep = state.forceString(*args[0], context, pos);
     state.forceList(*args[1], pos);
 
-    string res;
+    std::string res;
     res.reserve((args[1]->listSize() + 32) * sep.size());
     bool first = true;
 
@@ -3684,12 +3706,12 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
             .errPos = pos
         }));
 
-    vector<string> from;
+    std::vector<std::string> from;
     from.reserve(args[0]->listSize());
     for (auto elem : args[0]->listItems())
         from.emplace_back(state.forceString(*elem, pos));
 
-    vector<std::pair<string, PathSet>> to;
+    std::vector<std::pair<std::string, PathSet>> to;
     to.reserve(args[1]->listSize());
     for (auto elem : args[1]->listItems()) {
         PathSet ctx;
@@ -3700,7 +3722,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     PathSet context;
     auto s = state.forceString(*args[2], context, pos);
 
-    string res;
+    std::string res;
     // Loops one past last character to handle the case where 'from' contains an empty string.
     for (size_t p = 0; p <= s.size(); ) {
         bool found = false;
@@ -3841,7 +3863,7 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
         .name = name,
         .args = {},
         .arity = arity,
-        .fun = fun
+        .fun = fun,
     });
 }
 
@@ -3913,6 +3935,9 @@ void EvalState::createBaseEnv()
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
+            if (!primOp.experimentalFeature
+                || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature))
+            {
                 addPrimOp({
                     .fun = primOp.fun,
                     .arity = std::max(primOp.args.size(), primOp.arity),
@@ -3920,6 +3945,7 @@ void EvalState::createBaseEnv()
                     .args = primOp.args,
                     .doc = primOp.doc,
                 });
+            }
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */

src/libexpr/primops.hh

@@ -16,6 +16,7 @@ struct RegisterPrimOp
         size_t arity = 0;
         const char * doc;
         PrimOpFun fun;
+        std::optional<ExperimentalFeature> experimentalFeature;
     };
 
     typedef std::vector<Info> PrimOps;
@@ -35,6 +36,7 @@ struct RegisterPrimOp
 /* These primops are disabled without enableNativeCode, but plugins
    may wish to use them in limited contexts without globally enabling
    them. */
+
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
 

src/libexpr/primops/context.cc

@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "derivations.hh"
 #include "store-api.hh"
 
 namespace nix {
@@ -37,7 +38,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po
 
     PathSet context2;
     for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
+        context2.insert(p.at(0) == '=' ? std::string(p, 1) : p);
 
     v.mkString(*s, context2);
 }
@@ -76,14 +77,14 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
     auto contextInfos = std::map<Path, ContextInfo>();
     for (const auto & p : context) {
         Path drv;
-        string output;
+        std::string output;
         const Path * path = &p;
         if (p.at(0) == '=') {
-            drv = string(p, 1);
+            drv = std::string(p, 1);
             path = &drv;
         } else if (p.at(0) == '!') {
-            std::pair<string, string> ctx = decodeContext(p);
+            NixStringContextElem ctx = decodeContext(*state.store, p);
-            drv = ctx.first;
+            drv = state.store->printStorePath(ctx.first);
             output = ctx.second;
             path = &drv;
         }
@@ -166,7 +167,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                         .errPos = *i.pos
                     });
                 }
-                context.insert("=" + string(i.name));
+                context.insert("=" + std::string(i.name));
             }
         }
 

src/libexpr/primops/fetchClosure.cc

@@ -0,0 +1,161 @@
+#include "primops.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "url.hh"
+
+namespace nix {
+
+static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[0], pos);
+
+    std::optional<std::string> fromStoreUrl;
+    std::optional<StorePath> fromPath;
+    bool toCA = false;
+    std::optional<StorePath> toPath;
+
+    for (auto & attr : *args[0]->attrs) {
+        if (attr.name == "fromPath") {
+            PathSet context;
+            fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+        }
+
+        else if (attr.name == "toPath") {
+            state.forceValue(*attr.value, *attr.pos);
+            toCA = true;
+            if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+                PathSet context;
+                toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+            }
+        }
+
+        else if (attr.name == "fromStore")
+            fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
+
+        else
+            throw Error({
+                .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name),
+                .errPos = pos
+            });
+    }
+
+    if (!fromPath)
+        throw Error({
+            .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
+            .errPos = pos
+        });
+
+    if (!fromStoreUrl)
+        throw Error({
+            .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
+            .errPos = pos
+        });
+
+    auto parsedURL = parseURL(*fromStoreUrl);
+
+    if (parsedURL.scheme != "http" &&
+        parsedURL.scheme != "https" &&
+        !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
+        throw Error({
+            .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
+            .errPos = pos
+        });
+
+    if (!parsedURL.query.empty())
+        throw Error({
+            .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
+            .errPos = pos
+        });
+
+    auto fromStore = openStore(parsedURL.to_string());
+
+    if (toCA) {
+        if (!toPath || !state.store->isValidPath(*toPath)) {
+            auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
+            auto i = remappings.find(*fromPath);
+            assert(i != remappings.end());
+            if (toPath && *toPath != i->second)
+                throw Error({
+                    .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+                        state.store->printStorePath(*fromPath),
+                        state.store->printStorePath(i->second),
+                        state.store->printStorePath(*toPath)),
+                    .errPos = pos
+                });
+            if (!toPath)
+                throw Error({
+                    .msg = hintfmt(
+                        "rewriting '%s' to content-addressed form yielded '%s'; "
+                        "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+                        state.store->printStorePath(*fromPath),
+                        state.store->printStorePath(i->second)),
+                    .errPos = pos
+                });
+        }
+    } else {
+        if (!state.store->isValidPath(*fromPath))
+            copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+        toPath = fromPath;
+    }
+
+    /* In pure mode, require a CA path. */
+    if (evalSettings.pureEval) {
+        auto info = state.store->queryPathInfo(*toPath);
+        if (!info->isContentAddressed(*state.store))
+            throw Error({
+                .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
+                    state.store->printStorePath(*toPath)),
+                .errPos = pos
+            });
+    }
+
+    auto toPathS = state.store->printStorePath(*toPath);
+    v.mkString(toPathS, {toPathS});
+}
+
+static RegisterPrimOp primop_fetchClosure({
+    .name = "__fetchClosure",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a Nix store closure from a binary cache, rewriting it into
+      content-addressed form. For example,
+
+      ```nix
+      builtins.fetchClosure {
+        fromStore = "https://cache.nixos.org";
+        fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+        toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+      }
+      ```
+
+      fetches `/nix/store/r2jd...` from the specified binary cache,
+      and rewrites it into the content-addressed store path
+      `/nix/store/ldbh...`.
+
+      If `fromPath` is already content-addressed, or if you are
+      allowing impure evaluation (`--impure`), then `toPath` may be
+      omitted.
+
+      To find out the correct value for `toPath` given a `fromPath`,
+      you can use `nix store make-content-addressed`:
+
+      ```console
+      # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
+      rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
+      ```
+
+      This function is similar to `builtins.storePath` in that it
+      allows you to use a previously built store path in a Nix
+      expression. However, it is more reproducible because it requires
+      specifying a binary cache from which the path can be fetched.
+      Also, requiring a content-addressed final store path avoids the
+      need for users to configure binary cache public keys.
+
+      This function is only available if you enable the experimental
+      feature `fetch-closure`.
+    )",
+    .fun = prim_fetchClosure,
+    .experimentalFeature = Xp::FetchClosure,
+});
+
+}

src/libexpr/primops/fetchMercurial.cc

@@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     fetchers::Attrs attrs;
     attrs.insert_or_assign("type", "hg");
     attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
-    attrs.insert_or_assign("name", string(name));
+    attrs.insert_or_assign("name", std::string(name));
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
     auto input = fetchers::Input::fromAttrs(std::move(attrs));

src/libexpr/primops/fetchTree.cc

@@ -19,7 +19,7 @@ void emitTreeAttrs(
     bool emptyRevFallback,
     bool forceDirty)
 {
-    assert(input.isImmutable());
+    assert(input.isLocked());
 
     auto attrs = state.buildBindings(8);
 
@@ -166,8 +166,8 @@ static void fetchTree(
     if (!evalSettings.pureEval && !input.isDirect())
         input = lookupInRegistries(state.store, input).first;
 
-    if (evalSettings.pureEval && !input.isImmutable())
+    if (evalSettings.pureEval && !input.isLocked())
-        state.debug_throw(EvalError("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos));
+        state.debug_throw(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", pos));
 
     auto [tree, input2] = input.fetch(state.store);
 
@@ -186,7 +186,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
 static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
 
 static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
-    const string & who, bool unpack, std::string name)
+    const std::string & who, bool unpack, std::string name)
 {
     std::optional<std::string> url;
     std::optional<Hash> expectedHash;
@@ -198,7 +198,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
         state.forceAttrs(*args[0], pos);
 
         for (auto & attr : *args[0]->attrs) {
-            string n(attr.name);
+            std::string n(attr.name);
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "sha256")
@@ -230,6 +230,21 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     if (evalSettings.pureEval && !expectedHash)
         state.debug_throw(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who));
 
+    // early exit if pinned and already in the store
+    if (expectedHash && expectedHash->type == htSHA256) {
+        auto expectedPath =
+            unpack
+            ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {})
+            : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {});
+
+        if (state.store->isValidPath(expectedPath)) {
+            state.allowAndSetStorePathString(expectedPath, v);
+            return;
+        }
+    }
+
+    // TODO: fetching may fail, yet the path may be substitutable.
+    //       https://github.com/NixOS/nix/issues/4313
     auto storePath =
         unpack
         ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
@@ -244,10 +259,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
                 *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)));
     }
 
-    state.allowPath(storePath);
+    state.allowAndSetStorePathString(storePath, v);
-
-    auto path = state.store->printStorePath(storePath);
-    v.mkString(path, PathSet({path}));
 }
 
 static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -317,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({
     .fun = prim_fetchTarball,
 });
 
-static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
 }

src/libexpr/primops/fromTOML.cc

@@ -9,7 +9,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
 {
     auto toml = state.forceStringNoCtx(*args[0], pos);
 
-    std::istringstream tomlStream(string{toml});
+    std::istringstream tomlStream(std::string{toml});
 
     std::function<void(Value &, toml::value)> visit;
 

src/libexpr/symbol-table.hh

@@ -17,8 +17,8 @@ namespace nix {
 class Symbol
 {
 private:
-    const string * s; // pointer into SymbolTable
+    const std::string * s; // pointer into SymbolTable
-    Symbol(const string * s) : s(s) { };
+    Symbol(const std::string * s) : s(s) { };
     friend class SymbolTable;
 
 public:
@@ -72,7 +72,7 @@ class SymbolTable
 {
 private:
     std::unordered_map<std::string_view, Symbol> symbols;
-    std::list<string> store;
+    std::list<std::string> store;
 
 public:
     Symbol create(std::string_view s)
@@ -84,7 +84,7 @@ public:
         auto it = symbols.find(s);
         if (it != symbols.end()) return it->second;
 
-        const string & rawSym = store.emplace_back(s);
+        auto & rawSym = store.emplace_back(s);
         return symbols.emplace(rawSym, Symbol(&rawSym)).first->second;
     }
 

src/libexpr/value-to-xml.cc

@@ -9,7 +9,7 @@
 namespace nix {
 
 
-static XMLAttrs singletonAttrs(const string & name, const string & value)
+static XMLAttrs singletonAttrs(const std::string & name, const std::string & value)
 {
     XMLAttrs attrs;
     attrs[name] = value;

src/libexpr/value.hh

@@ -57,6 +57,8 @@ struct ExprLambda;
 struct PrimOp;
 class Symbol;
 struct Pos;
+class StorePath;
+class Store;
 class EvalState;
 class XMLWriter;
 class JSONPlaceholder;
@@ -64,6 +66,8 @@ class JSONPlaceholder;
 
 typedef int64_t NixInt;
 typedef double NixFloat;
+typedef std::pair<StorePath, std::string> NixStringContextElem;
+typedef std::vector<NixStringContextElem> NixStringContext;
 
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
@@ -77,20 +81,20 @@ class ExternalValueBase
 
     public:
     /* Return a simple string describing the type */
-    virtual string showType() const = 0;
+    virtual std::string showType() const = 0;
 
     /* Return a string to be used in builtins.typeOf */
-    virtual string typeOf() const = 0;
+    virtual std::string typeOf() const = 0;
 
     /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
-     * error
+     * error.
      */
-    virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
+    virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
 
     /* Compare to another value of the same type. Defaults to uncomparable,
      * i.e. always false.
      */
-    virtual bool operator==(const ExternalValueBase & b) const;
+    virtual bool operator ==(const ExternalValueBase & b) const;
 
     /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
     virtual void printValueAsJSON(EvalState & state, bool strict,
@@ -114,11 +118,14 @@ struct Value
 private:
     InternalType internalType;
 
-friend std::string showType(const Value & v);
+    friend std::string showType(const Value & v);
-friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
+
+    void print(std::ostream & str, std::set<const void *> * seen) const;
 
 public:
 
+    void print(std::ostream & str, bool showRepeated = false) const;
+
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
     // needed by callers into methods of this type
@@ -368,7 +375,7 @@ public:
        non-trivial. */
     bool isTrivial() const;
 
-    std::vector<std::pair<Path, std::string>> getContext();
+    NixStringContext getContext(const Store &);
 
     auto listItems()
     {

src/libfetchers/cache.cc

@@ -52,13 +52,13 @@ struct CacheImpl : Cache
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
-        bool immutable) override
+        bool locked) override
     {
         _state.lock()->add.use()
             (attrsToJSON(inAttrs).dump())
             (attrsToJSON(infoAttrs).dump())
             (store->printStorePath(storePath))
-            (immutable)
+            (locked)
             (time(0)).exec();
     }
 
@@ -91,7 +91,7 @@ struct CacheImpl : Cache
 
         auto infoJSON = stmt.getStr(0);
         auto storePath = store->parseStorePath(stmt.getStr(1));
-        auto immutable = stmt.getInt(2) != 0;
+        auto locked = stmt.getInt(2) != 0;
         auto timestamp = stmt.getInt(3);
 
         store->addTempRoot(storePath);
@@ -105,7 +105,7 @@ struct CacheImpl : Cache
             inAttrsJSON, infoJSON, store->printStorePath(storePath));
 
         return Result {
-            .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
             .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
             .storePath = std::move(storePath)
         };

src/libfetchers/cache.hh

@@ -13,7 +13,7 @@ struct Cache
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
-        bool immutable) = 0;
+        bool locked) = 0;
 
     virtual std::optional<std::pair<Attrs, StorePath>> lookup(
         ref<Store> store,

src/libfetchers/fetch-settings.cc

@@ -0,0 +1,13 @@
+#include "fetch-settings.hh"
+
+namespace nix {
+
+FetchSettings::FetchSettings()
+{
+}
+
+FetchSettings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
+}

src/libfetchers/fetch-settings.hh

@@ -0,0 +1,93 @@
+#pragma once
+
+#include "types.hh"
+#include "config.hh"
+#include "util.hh"
+
+#include <map>
+#include <limits>
+
+#include <sys/types.h>
+
+namespace nix {
+
+struct FetchSettings : public Config
+{
+    FetchSettings();
+
+    Setting<StringMap> accessTokens{this, {}, "access-tokens",
+        R"(
+          Access tokens used to access protected GitHub, GitLab, or
+          other locations requiring token-based authentication.
+
+          Access tokens are specified as a string made up of
+          space-separated `host=token` values.  The specific token
+          used is selected by matching the `host` portion against the
+          "host" specification of the input. The actual use of the
+          `token` value is determined by the type of resource being
+          accessed:
+
+          * Github: the token value is the OAUTH-TOKEN string obtained
+            as the Personal Access Token from the Github server (see
+            https://docs.github.com/en/developers/apps/building-oauth-apps/authorizing-oauth-apps).
+
+          * Gitlab: the token value is either the OAuth2 token or the
+            Personal Access Token (these are different types tokens
+            for gitlab, see
+            https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
+            The `token` value should be `type:tokenstring` where
+            `type` is either `OAuth2` or `PAT` to indicate which type
+            of token is being specified.
+
+          Example `~/.config/nix/nix.conf`:
+
+          ```
+          access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk
+          ```
+
+          Example `~/code/flake.nix`:
+
+          ```nix
+          input.foo = {
+            type = "gitlab";
+            host = "gitlab.mycompany.com";
+            owner = "mycompany";
+            repo = "pro";
+          };
+          ```
+
+          This example specifies three tokens, one each for accessing
+          github.com, gitlab.mycompany.com, and sourceforge.net.
+
+          The `input.foo` uses the "gitlab" fetcher, which might
+          requires specifying the token type along with the token
+          value.
+          )"};
+
+    Setting<bool> allowDirty{this, true, "allow-dirty",
+        "Whether to allow dirty Git/Mercurial trees."};
+
+    Setting<bool> warnDirty{this, true, "warn-dirty",
+        "Whether to warn about dirty Git/Mercurial trees."};
+
+    Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
+        "Path or URI of the global flake registry."};
+
+    Setting<bool> useRegistries{this, true, "use-registries",
+        "Whether to use flake registries to resolve flake references."};
+
+    Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config",
+        "Whether to accept nix configuration from a flake without prompting."};
+
+    Setting<std::string> commitLockFileSummary{
+        this, "", "commit-lockfile-summary",
+        R"(
+          The commit summary to use when committing changed flake lock files. If
+          empty, the summary is generated based on the action performed.
+        )"};
+};
+
+// FIXME: don't use a global variable.
+extern FetchSettings fetchSettings;
+
+}

src/libfetchers/fetchers.cc

@@ -24,11 +24,11 @@ static void fixupInput(Input & input)
     input.getType();
     input.getRef();
     if (input.getRev())
-        input.immutable = true;
+        input.locked = true;
     input.getRevCount();
     input.getLastModified();
     if (input.getNarHash())
-        input.immutable = true;
+        input.locked = true;
 }
 
 Input Input::fromURL(const ParsedURL & url)
@@ -124,15 +124,13 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
             debug("using substituted/cached input '%s' in '%s'",
                 to_string(), store->printStorePath(storePath));
 
-            auto actualPath = store->toRealPath(storePath);
+            return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this};
-
-            return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this};
         } catch (Error & e) {
             debug("substitution of input '%s' failed: %s", to_string(), e.what());
         }
     }
 
-    auto [tree, input] = [&]() -> std::pair<Tree, Input> {
+    auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
         try {
             return scheme->fetch(store, *this);
         } catch (Error & e) {
@@ -141,8 +139,10 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
         }
     }();
 
-    if (tree.actualPath == "")
+    Tree tree {
-        tree.actualPath = store->toRealPath(tree.storePath);
+        .actualPath = store->toRealPath(storePath),
+        .storePath = storePath,
+    };
 
     auto narHash = store->queryPathInfo(tree.storePath)->narHash;
     input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
@@ -165,7 +165,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
                 input.to_string(), *prevRevCount);
     }
 
-    input.immutable = true;
+    input.locked = true;
 
     assert(input.hasAllInfo());
 
@@ -209,7 +209,7 @@ StorePath Input::computeStorePath(Store & store) const
 {
     auto narHash = getNarHash();
     if (!narHash)
-        throw Error("cannot compute store path for mutable input '%s'", to_string());
+        throw Error("cannot compute store path for unlocked input '%s'", to_string());
     return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
 }
 

src/libfetchers/fetchers.hh

@@ -16,7 +16,6 @@ struct Tree
 {
     Path actualPath;
     StorePath storePath;
-    Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {}
 };
 
 struct InputScheme;
@@ -35,7 +34,7 @@ struct Input
 
     std::shared_ptr<InputScheme> scheme; // note: can be null
     Attrs attrs;
-    bool immutable = false;
+    bool locked = false;
     bool direct = true;
 
     /* path of the parent of this input, used for relative path resolution */
@@ -60,9 +59,9 @@ public:
        one that goes through a registry. */
     bool isDirect() const { return direct; }
 
-    /* Check whether this is an "immutable" input, that is,
+    /* Check whether this is a "locked" input, that is,
        one that contains a commit hash or content hash. */
-    bool isImmutable() const { return immutable; }
+    bool isLocked() const { return locked; }
 
     bool hasAllInfo() const;
 
@@ -70,6 +69,8 @@ public:
 
     bool contains(const Input & other) const;
 
+    /* Fetch the input into the Nix store, returning the location in
+       the Nix store and the locked input. */
     std::pair<Tree, Input> fetch(ref<Store> store) const;
 
     Input applyOverrides(
@@ -131,7 +132,7 @@ struct InputScheme
 
     virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
 
-    virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
+    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
@@ -147,14 +148,14 @@ DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable,
+    bool locked,
     const Headers & headers = {});
 
 std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable,
+    bool locked,
     const Headers & headers = {});
 
 }

src/libfetchers/git.cc

@@ -6,6 +6,8 @@
 #include "url-parts.hh"
 #include "pathlocks.hh"
 
+#include "fetch-settings.hh"
+
 #include <sys/time.h>
 #include <sys/wait.h>
 
@@ -172,7 +174,7 @@ struct GitInputScheme : InputScheme
         return {isLocal, isLocal ? url.path : url.base};
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
         Input input(_input);
 
@@ -187,7 +189,7 @@ struct GitInputScheme : InputScheme
         if (submodules) cacheType += "-submodules";
         if (allRefs) cacheType += "-all-refs";
 
-        auto getImmutableAttrs = [&]()
+        auto getLockedAttrs = [&]()
         {
             return Attrs({
                 {"type", cacheType},
@@ -197,21 +199,18 @@ struct GitInputScheme : InputScheme
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, Input>
+            -> std::pair<StorePath, Input>
         {
             assert(input.getRev());
             assert(!_input.getRev() || _input.getRev() == input.getRev());
             if (!shallow)
                 input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
             input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
-            return {
+            return {std::move(storePath), input};
-                Tree(store->toRealPath(storePath), std::move(storePath)),
-                input
-            };
         };
 
         if (input.getRev()) {
-            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            if (auto res = getCache()->lookup(store, getLockedAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
@@ -223,22 +222,46 @@ struct GitInputScheme : InputScheme
         if (!input.getRef() && !input.getRev() && isLocal) {
             bool clean = false;
 
-            /* Check whether this repo has any commits. There are
+            auto env = getEnv();
-               probably better ways to do this. */
+            // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
-            auto gitDir = actualUrl + "/.git";
+            // that way unknown errors can lead to a failure instead of continuing through the wrong code path
-            auto commonGitDir = chomp(runProgram(
+            env["LC_ALL"] = "C";
-                "git",
-                true,
-                { "-C", actualUrl, "rev-parse", "--git-common-dir" }
-            ));
-            if (commonGitDir != ".git")
-                    gitDir = commonGitDir;
 
-            bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
+            /* Check whether HEAD points to something that looks like a commit,
+               since that is the refrence we want to use later on. */
+            auto result = runProgram(RunOptions {
+                .program = "git",
+                .args = { "-C", actualUrl, "--git-dir=.git", "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
+                .environment = env,
+                .mergeStderrToStdout = true
+            });
+            auto exitCode = WEXITSTATUS(result.first);
+            auto errorMessage = result.second;
 
+            if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
+                throw Error("'%s' is not a Git repository", actualUrl);
+            } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
+                // indicates that the repo does not have any commits
+                // we want to proceed and will consider it dirty later
+            } else if (exitCode != 0) {
+                // any other errors should lead to a failure
+                throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage);
+            }
+
+            bool hasHead = exitCode == 0;
             try {
-                if (haveCommits) {
+                if (hasHead) {
-                    runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
+                    // Using git diff is preferrable over lower-level operations here,
+                    // because its conceptually simpler and we only need the exit code anyways.
+                    auto gitDiffOpts = Strings({ "-C", actualUrl, "diff", "HEAD", "--quiet"});
+                    if (!submodules) {
+                        // Changes in submodules should only make the tree dirty
+                        // when those submodules will be copied as well.
+                        gitDiffOpts.emplace_back("--ignore-submodules");
+                    }
+                    gitDiffOpts.emplace_back("--");
+                    runProgram("git", true, gitDiffOpts);
+
                     clean = true;
                 }
             } catch (ExecError & e) {
@@ -249,10 +272,10 @@ struct GitInputScheme : InputScheme
 
                 /* This is an unclean working tree. So copy all tracked files. */
 
-                if (!settings.allowDirty)
+                if (!fetchSettings.allowDirty)
                     throw Error("Git tree '%s' is dirty", actualUrl);
 
-                if (settings.warnDirty)
+                if (fetchSettings.warnDirty)
                     warn("Git tree '%s' is dirty", actualUrl);
 
                 auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" });
@@ -262,9 +285,11 @@ struct GitInputScheme : InputScheme
                 auto files = tokenizeString<std::set<std::string>>(
                     runProgram("git", true, gitOpts), "\0"s);
 
+                Path actualPath(absPath(actualUrl));
+
                 PathFilter filter = [&](const Path & p) -> bool {
-                    assert(hasPrefix(p, actualUrl));
+                    assert(hasPrefix(p, actualPath));
-                    std::string file(p, actualUrl.size() + 1);
+                    std::string file(p, actualPath.size() + 1);
 
                     auto st = lstat(p);
 
@@ -277,24 +302,21 @@ struct GitInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 // FIXME: maybe we should use the timestamp of the last
                 // modified dirty file?
                 input.attrs.insert_or_assign(
                     "lastModified",
-                    haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
+                    hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
 
-                return {
+                return {std::move(storePath), input};
-                    Tree(store->toRealPath(storePath), std::move(storePath)),
-                    input
-                };
             }
         }
 
         if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
 
-        Attrs mutableAttrs({
+        Attrs unlockedAttrs({
             {"type", cacheType},
             {"name", name},
             {"url", actualUrl},
@@ -313,7 +335,7 @@ struct GitInputScheme : InputScheme
 
         } else {
 
-            if (auto res = getCache()->lookup(store, mutableAttrs)) {
+            if (auto res = getCache()->lookup(store, unlockedAttrs)) {
                 auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
                 if (!input.getRev() || input.getRev() == rev2) {
                     input.attrs.insert_or_assign("rev", rev2.gitRev());
@@ -410,7 +432,7 @@ struct GitInputScheme : InputScheme
 
         /* Now that we know the ref, check again whether we have it in
            the store. */
-        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+        if (auto res = getCache()->lookup(store, getLockedAttrs()))
             return makeResult(res->first, std::move(res->second));
 
         Path tmpDir = createTempDir();
@@ -482,14 +504,14 @@ struct GitInputScheme : InputScheme
         if (!_input.getRev())
             getCache()->add(
                 store,
-                mutableAttrs,
+                unlockedAttrs,
                 infoAttrs,
                 storePath,
                 false);
 
         getCache()->add(
             store,
-            getImmutableAttrs(),
+            getLockedAttrs(),
             infoAttrs,
             storePath,
             true);

src/libfetchers/github.cc

@@ -1,13 +1,16 @@
 #include "filetransfer.hh"
 #include "cache.hh"
-#include "fetchers.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "types.hh"
 #include "url-parts.hh"
 
+#include "fetchers.hh"
+#include "fetch-settings.hh"
+
 #include <optional>
 #include <nlohmann/json.hpp>
+#include <fstream>
 
 namespace nix::fetchers {
 
@@ -17,7 +20,7 @@ struct DownloadUrl
     Headers headers;
 };
 
-// A github or gitlab host
+// A github, gitlab, or sourcehut host
 const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
 std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
@@ -156,7 +159,7 @@ struct GitArchiveInputScheme : InputScheme
 
     std::optional<std::string> getAccessToken(const std::string & host) const
     {
-        auto tokens = settings.accessTokens.get();
+        auto tokens = fetchSettings.accessTokens.get();
         if (auto token = get(tokens, host))
             return *token;
         return {};
@@ -180,7 +183,7 @@ struct GitArchiveInputScheme : InputScheme
 
     virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
         Input input(_input);
 
@@ -192,17 +195,14 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.erase("ref");
         input.attrs.insert_or_assign("rev", rev->gitRev());
 
-        Attrs immutableAttrs({
+        Attrs lockedAttrs({
             {"type", "git-tarball"},
             {"rev", rev->gitRev()},
         });
 
-        if (auto res = getCache()->lookup(store, immutableAttrs)) {
+        if (auto res = getCache()->lookup(store, lockedAttrs)) {
             input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
-            return {
+            return {std::move(res->second), input};
-                Tree(store->toRealPath(res->second), std::move(res->second)),
-                input
-            };
         }
 
         auto url = getDownloadUrl(input);
@@ -213,7 +213,7 @@ struct GitArchiveInputScheme : InputScheme
 
         getCache()->add(
             store,
-            immutableAttrs,
+            lockedAttrs,
             {
                 {"rev", rev->gitRev()},
                 {"lastModified", uint64_t(lastModified)}
@@ -221,7 +221,7 @@ struct GitArchiveInputScheme : InputScheme
             tree.storePath,
             true);
 
-        return {std::move(tree), input};
+        return {std::move(tree.storePath), input};
     }
 };
 
@@ -348,7 +348,97 @@ struct GitLabInputScheme : GitArchiveInputScheme
     }
 };
 
+struct SourceHutInputScheme : GitArchiveInputScheme
+{
+    std::string type() override { return "sourcehut"; }
+
+    std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
+    {
+        // SourceHut supports both PAT and OAuth2. See
+        // https://man.sr.ht/meta.sr.ht/oauth.md
+        return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
+        // Note: This currently serves no purpose, as this kind of authorization
+        // does not allow for downloading tarballs on sourcehut private repos.
+        // Once it is implemented, however, should work as expected.
+    }
+
+    Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
+    {
+        // TODO: In the future, when the sourcehut graphql API is implemented for mercurial
+        // and with anonymous access, this method should use it instead.
+
+        auto ref = *input.getRef();
+
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
+        auto base_url = fmt("https://%s/%s/%s",
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
+
+        Headers headers = makeHeadersWithAuthTokens(host);
+
+        std::string ref_uri;
+        if (ref == "HEAD") {
+            auto file = store->toRealPath(
+                downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath);
+            std::ifstream is(file);
+            std::string line;
+            getline(is, line);
+
+            auto ref_index = line.find("ref: ");
+            if (ref_index == std::string::npos) {
+                throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref);
+            }
+
+            ref_uri = line.substr(ref_index+5, line.length()-1);
+        } else
+            ref_uri = fmt("refs/(heads|tags)/%s", ref);
+
+        auto file = store->toRealPath(
+            downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
+        std::ifstream is(file);
+
+        std::string line;
+        std::string id;
+        while(getline(is, line)) {
+            // Append $ to avoid partial name matches
+            std::regex pattern(fmt("%s$", ref_uri));
+
+            if (std::regex_search(line, pattern)) {
+                id = line.substr(0, line.find('\t'));
+                break;
+            }
+        }
+
+        if(id.empty())
+            throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
+
+        auto rev = Hash::parseAny(id, htSHA1);
+        debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
+        return rev;
+    }
+
+    DownloadUrl getDownloadUrl(const Input & input) const override
+    {
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
+        auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz",
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            input.getRev()->to_string(Base16, false));
+
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl { url, headers };
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
+        Input::fromURL(fmt("git+https://%s/%s/%s",
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+            .applyOverrides(input.getRef(), input.getRev())
+            .clone(destDir);
+    }
+};
+
 static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
 static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
+static auto rSourceHutInputScheme = OnStartup([] { registerInputScheme(std::make_unique<SourceHutInputScheme>()); });
 
 }

src/libfetchers/indirect.cc

@@ -94,7 +94,7 @@ struct IndirectInputScheme : InputScheme
         return input;
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
     {
         throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
     }

src/libfetchers/mercurial.cc

@@ -5,6 +5,8 @@
 #include "store-api.hh"
 #include "url-parts.hh"
 
+#include "fetch-settings.hh"
+
 #include <sys/time.h>
 
 using namespace std::string_literals;
@@ -26,7 +28,7 @@ static RunOptions hgOptions(const Strings & args)
 }
 
 // runProgram wrapper that uses hgOptions instead of stock RunOptions.
-static string runHg(const Strings & args, const std::optional<std::string> & input = {})
+static std::string runHg(const Strings & args, const std::optional<std::string> & input = {})
 {
     RunOptions opts = hgOptions(args);
     opts.input = input;
@@ -143,7 +145,7 @@ struct MercurialInputScheme : InputScheme
         return {isLocal, isLocal ? url.path : url.base};
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
         Input input(_input);
 
@@ -165,10 +167,10 @@ struct MercurialInputScheme : InputScheme
                 /* This is an unclean working tree. So copy all tracked
                    files. */
 
-                if (!settings.allowDirty)
+                if (!fetchSettings.allowDirty)
                     throw Error("Mercurial tree '%s' is unclean", actualUrl);
 
-                if (settings.warnDirty)
+                if (fetchSettings.warnDirty)
                     warn("Mercurial tree '%s' is unclean", actualUrl);
 
                 input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
@@ -193,16 +195,13 @@ struct MercurialInputScheme : InputScheme
 
                 auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
-                return {
+                return {std::move(storePath), input};
-                    Tree(store->toRealPath(storePath), std::move(storePath)),
-                    input
-                };
             }
         }
 
         if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
 
-        auto getImmutableAttrs = [&]()
+        auto getLockedAttrs = [&]()
         {
             return Attrs({
                 {"type", "hg"},
@@ -212,32 +211,29 @@ struct MercurialInputScheme : InputScheme
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, Input>
+            -> std::pair<StorePath, Input>
         {
             assert(input.getRev());
             assert(!_input.getRev() || _input.getRev() == input.getRev());
             input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
-            return {
+            return {std::move(storePath), input};
-                Tree(store->toRealPath(storePath), std::move(storePath)),
-                input
-            };
         };
 
         if (input.getRev()) {
-            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            if (auto res = getCache()->lookup(store, getLockedAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
         auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
 
-        Attrs mutableAttrs({
+        Attrs unlockedAttrs({
             {"type", "hg"},
             {"name", name},
             {"url", actualUrl},
             {"ref", *input.getRef()},
         });
 
-        if (auto res = getCache()->lookup(store, mutableAttrs)) {
+        if (auto res = getCache()->lookup(store, unlockedAttrs)) {
             auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
             if (!input.getRev() || input.getRev() == rev2) {
                 input.attrs.insert_or_assign("rev", rev2.gitRev());
@@ -260,7 +256,7 @@ struct MercurialInputScheme : InputScheme
                     runHg({ "pull", "-R", cacheDir, "--", actualUrl });
                 }
                 catch (ExecError & e) {
-                    string transJournal = cacheDir + "/.hg/store/journal";
+                    auto transJournal = cacheDir + "/.hg/store/journal";
                     /* hg throws "abandoned transaction" error only if this file exists */
                     if (pathExists(transJournal)) {
                         runHg({ "recover", "-R", cacheDir });
@@ -283,7 +279,7 @@ struct MercurialInputScheme : InputScheme
         auto revCount = std::stoull(tokens[1]);
         input.attrs.insert_or_assign("ref", tokens[2]);
 
-        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+        if (auto res = getCache()->lookup(store, getLockedAttrs()))
             return makeResult(res->first, std::move(res->second));
 
         Path tmpDir = createTempDir();
@@ -303,14 +299,14 @@ struct MercurialInputScheme : InputScheme
         if (!_input.getRev())
             getCache()->add(
                 store,
-                mutableAttrs,
+                unlockedAttrs,
                 infoAttrs,
                 storePath,
                 false);
 
         getCache()->add(
             store,
-            getImmutableAttrs(),
+            getLockedAttrs(),
             infoAttrs,
             storePath,
             true);

src/libfetchers/path.cc

@@ -1,5 +1,6 @@
 #include "fetchers.hh"
 #include "store-api.hh"
+#include "archive.hh"
 
 namespace nix::fetchers {
 
@@ -80,8 +81,9 @@ struct PathInputScheme : InputScheme
         // nothing to do
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
+        Input input(_input);
         std::string absPath;
         auto path = getStrAttr(input.attrs, "path");
 
@@ -111,14 +113,17 @@ struct PathInputScheme : InputScheme
         if (storePath)
             store->addTempRoot(*storePath);
 
-        if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
+        time_t mtime = 0;
+        if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
             // FIXME: try to substitute storePath.
-            storePath = store->addToStore("source", absPath);
+            auto src = sinkToSource([&](Sink & sink) {
+                mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter);
+            });
+            storePath = store->addToStoreFromDump(*src, "source");
+        }
+        input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
 
-        return {
+        return {std::move(*storePath), input};
-            Tree(store->toRealPath(*storePath), std::move(*storePath)),
-            input
-        };
     }
 };
 

src/libfetchers/registry.cc

@@ -5,6 +5,8 @@
 #include "store-api.hh"
 #include "local-fs-store.hh"
 
+#include "fetch-settings.hh"
+
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
@@ -150,7 +152,7 @@ void overrideRegistry(
 static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
 {
     static auto reg = [&]() {
-        auto path = settings.flakeRegistry.get();
+        auto path = fetchSettings.flakeRegistry.get();
 
         if (!hasPrefix(path, "/")) {
             auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;

src/libfetchers/tarball.cc

@@ -13,7 +13,7 @@ DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable,
+    bool locked,
     const Headers & headers)
 {
     // FIXME: check store
@@ -88,7 +88,7 @@ DownloadFileResult downloadFile(
         inAttrs,
         infoAttrs,
         *storePath,
-        immutable);
+        locked);
 
     if (url != res.effectiveUri)
         getCache()->add(
@@ -100,7 +100,7 @@ DownloadFileResult downloadFile(
             },
             infoAttrs,
             *storePath,
-            immutable);
+            locked);
 
     return {
         .storePath = std::move(*storePath),
@@ -113,7 +113,7 @@ std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable,
+    bool locked,
     const Headers & headers)
 {
     Attrs inAttrs({
@@ -126,11 +126,11 @@ std::pair<Tree, time_t> downloadTarball(
 
     if (cached && !cached->expired)
         return {
-            Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
+            Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
             getIntAttr(cached->infoAttrs, "lastModified")
         };
 
-    auto res = downloadFile(store, url, name, immutable, headers);
+    auto res = downloadFile(store, url, name, locked, headers);
 
     std::optional<StorePath> unpackedStorePath;
     time_t lastModified;
@@ -160,10 +160,10 @@ std::pair<Tree, time_t> downloadTarball(
         inAttrs,
         infoAttrs,
         *unpackedStorePath,
-        immutable);
+        locked);
 
     return {
-        Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
+        Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
         lastModified,
     };
 }
@@ -202,7 +202,7 @@ struct TarballInputScheme : InputScheme
 
         Input input;
         input.attrs = attrs;
-        //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
+        //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
         return input;
     }
 
@@ -225,10 +225,10 @@ struct TarballInputScheme : InputScheme
         return true;
     }
 
-    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
     {
         auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
-        return {std::move(tree), input};
+        return {std::move(tree.storePath), input};
     }
 };
 

src/libmain/common-args.cc

@@ -4,7 +4,7 @@
 
 namespace nix {
 
-MixCommonArgs::MixCommonArgs(const string & programName)
+MixCommonArgs::MixCommonArgs(const std::string & programName)
     : programName(programName)
 {
     addFlag({

src/libmain/common-args.hh

@@ -11,8 +11,8 @@ class MixCommonArgs : public virtual Args
 {
     void initialFlagsProcessed() override;
 public:
-    string programName;
+    std::string programName;
-    MixCommonArgs(const string & programName);
+    MixCommonArgs(const std::string & programName);
 protected:
     virtual void pluginsInited() {}
 };

src/libmain/shared.cc

@@ -1,6 +1,7 @@
 #include "globals.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "gc-store.hh"
 #include "util.hh"
 #include "loggers.hh"
 
@@ -94,7 +95,7 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
 }
 
 
-string getArg(const string & opt,
+std::string getArg(const std::string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
@@ -322,14 +323,14 @@ void parseCmdLine(int argc, char * * argv,
 }
 
 
-void parseCmdLine(const string & programName, const Strings & args,
+void parseCmdLine(const std::string & programName, const Strings & args,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
 {
     LegacyArgs(programName, parseArg).parseCmdline(args);
 }
 
 
-void printVersion(const string & programName)
+void printVersion(const std::string & programName)
 {
     std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
     if (verbosity > lvlInfo) {
@@ -352,7 +353,7 @@ void printVersion(const string & programName)
 }
 
 
-void showManPage(const string & name)
+void showManPage(const std::string & name)
 {
     restoreProcessContext();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
@@ -361,13 +362,13 @@ void showManPage(const string & name)
 }
 
 
-int handleExceptions(const string & programName, std::function<void()> fun)
+int handleExceptions(const std::string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
 
     ErrorInfo::programName = baseNameOf(programName);
 
-    string error = ANSI_RED "error:" ANSI_NORMAL " ";
+    std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
             fun();
@@ -407,7 +408,7 @@ RunPager::RunPager()
     if (!isatty(STDOUT_FILENO)) return;
     char * pager = getenv("NIX_PAGER");
     if (!pager) pager = getenv("PAGER");
-    if (pager && ((string) pager == "" || (string) pager == "cat")) return;
+    if (pager && ((std::string) pager == "" || (std::string) pager == "cat")) return;
 
     Pipe toPager;
     toPager.create();

src/libmain/shared.hh

@@ -22,7 +22,7 @@ public:
     virtual ~Exit();
 };
 
-int handleExceptions(const string & programName, std::function<void()> fun);
+int handleExceptions(const std::string & programName, std::function<void()> fun);
 
 /* Don't forget to call initPlugins() after settings are initialized! */
 void initNix();
@@ -30,10 +30,10 @@ void initNix();
 void parseCmdLine(int argc, char * * argv,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
 
-void parseCmdLine(const string & programName, const Strings & args,
+void parseCmdLine(const std::string & programName, const Strings & args,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
 
-void printVersion(const string & programName);
+void printVersion(const std::string & programName);
 
 /* Ugh.  No better place to put this. */
 void printGCWarning();
@@ -50,10 +50,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     const StorePathSet & willSubstitute, const StorePathSet & unknown,
     uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
 
-string getArg(const string & opt,
+std::string getArg(const std::string & opt,
     Strings::iterator & i, const Strings::iterator & end);
 
-template<class N> N getIntArg(const string & opt,
+template<class N> N getIntArg(const std::string & opt,
     Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
 {
     ++i;
@@ -76,7 +76,7 @@ struct LegacyArgs : public MixCommonArgs
 
 
 /* Show the manual page for the specified program. */
-void showManPage(const string & name);
+void showManPage(const std::string & name);
 
 /* The constructor of this class starts a pager if stdout is a
    terminal and $PAGER is set. Stdout is redirected to the pager. */
@@ -96,7 +96,7 @@ extern volatile ::sig_atomic_t blockInt;
 
 /* GC helpers. */
 
-string showBytes(uint64_t bytes);
+std::string showBytes(uint64_t bytes);
 
 struct GCResults;
 

src/libstore/binary-cache-store.cc

@@ -307,7 +307,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     }});
 }
 
-StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
+StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
     FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
     if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
@@ -385,8 +385,14 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
         }});
 }
 
-StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
+StorePath BinaryCacheStore::addToStore(
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references)
+    std::string_view name,
+    const Path & srcPath,
+    FileIngestionMethod method,
+    HashType hashAlgo,
+    PathFilter & filter,
+    RepairFlag repair,
+    const StorePathSet & references)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
@@ -418,8 +424,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
     })->path;
 }
 
-StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
+StorePath BinaryCacheStore::addTextToStore(
-    const StorePathSet & references, RepairFlag repair)
+    std::string_view name,
+    std::string_view s,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     auto textHash = hashString(htSHA256, s);
     auto path = makeTextPath(name, textHash, references);

src/libstore/binary-cache-store.hh

@@ -2,6 +2,7 @@
 
 #include "crypto.hh"
 #include "store-api.hh"
+#include "log-store.hh"
 
 #include "pool.hh"
 
@@ -28,7 +29,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
         "other than -1 which we reserve to indicate Nix defaults should be used"};
 };
 
-class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
+class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
+    public virtual Store,
+    public virtual LogStore
 {
 
 private:
@@ -98,15 +101,23 @@ public:
     void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStoreFromDump(Source & dump, const string & name,
+    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override;
+        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
 
-    StorePath addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(
-        FileIngestionMethod method, HashType hashAlgo,
+        std::string_view name,
-        PathFilter & filter, RepairFlag repair, const StorePathSet & references) override;
+        const Path & srcPath,
+        FileIngestionMethod method,
+        HashType hashAlgo,
+        PathFilter & filter,
+        RepairFlag repair,
+        const StorePathSet & references) override;
 
-    StorePath addTextToStore(const string & name, const string & s,
+    StorePath addTextToStore(
-        const StorePathSet & references, RepairFlag repair) override;
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references,
+        RepairFlag repair) override;
 
     void registerDrvOutput(const Realisation & info) override;
 

src/libstore/build-result.hh

@@ -0,0 +1,90 @@
+#pragma once
+
+#include "realisation.hh"
+#include "derived-path.hh"
+
+#include <string>
+#include <chrono>
+
+
+namespace nix {
+
+struct BuildResult
+{
+    /* Note: don't remove status codes, and only add new status codes
+       at the end of the list, to prevent client/server
+       incompatibilities in the nix-store --serve protocol. */
+    enum Status {
+        Built = 0,
+        Substituted,
+        AlreadyValid,
+        PermanentFailure,
+        InputRejected,
+        OutputRejected,
+        TransientFailure, // possibly transient
+        CachedFailure, // no longer used
+        TimedOut,
+        MiscFailure,
+        DependencyFailed,
+        LogLimitExceeded,
+        NotDeterministic,
+        ResolvesToAlreadyValid,
+        NoSubstituters,
+    } status = MiscFailure;
+    std::string errorMsg;
+
+    std::string toString() const {
+        auto strStatus = [&]() {
+            switch (status) {
+                case Built: return "Built";
+                case Substituted: return "Substituted";
+                case AlreadyValid: return "AlreadyValid";
+                case PermanentFailure: return "PermanentFailure";
+                case InputRejected: return "InputRejected";
+                case OutputRejected: return "OutputRejected";
+                case TransientFailure: return "TransientFailure";
+                case CachedFailure: return "CachedFailure";
+                case TimedOut: return "TimedOut";
+                case MiscFailure: return "MiscFailure";
+                case DependencyFailed: return "DependencyFailed";
+                case LogLimitExceeded: return "LogLimitExceeded";
+                case NotDeterministic: return "NotDeterministic";
+                case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid";
+                default: return "Unknown";
+            };
+        }();
+        return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
+    }
+
+    /* How many times this build was performed. */
+    unsigned int timesBuilt = 0;
+
+    /* If timesBuilt > 1, whether some builds did not produce the same
+       result. (Note that 'isNonDeterministic = false' does not mean
+       the build is deterministic, just that we don't have evidence of
+       non-determinism.) */
+    bool isNonDeterministic = false;
+
+    /* The derivation we built or the store path we substituted. */
+    DerivedPath path;
+
+    /* For derivations, a mapping from the names of the wanted outputs
+       to actual paths. */
+    DrvOutputs builtOutputs;
+
+    /* The start/stop times of the build (or one of the rounds, if it
+       was repeated). */
+    time_t startTime = 0, stopTime = 0;
+
+    bool success()
+    {
+        return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;
+    }
+
+    void rethrow()
+    {
+        throw Error("%s", errorMsg);
+    }
+};
+
+}

src/libstore/build/derivation-goal.cc

@@ -66,7 +66,7 @@ namespace nix {
 
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
     const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
+    : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
     , useDerivation(true)
     , drvPath(drvPath)
     , wantedOutputs(wantedOutputs)
@@ -85,7 +85,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
 
 DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
     const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker)
+    : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
     , useDerivation(false)
     , drvPath(drvPath)
     , wantedOutputs(wantedOutputs)
@@ -116,7 +116,7 @@ DerivationGoal::~DerivationGoal()
 }
 
 
-string DerivationGoal::key()
+std::string DerivationGoal::key()
 {
     /* Ensure that derivations get built in order of their name,
        i.e. a derivation named "aardvark" always comes before
@@ -135,7 +135,7 @@ void DerivationGoal::killChild()
 void DerivationGoal::timedOut(Error && ex)
 {
     killChild();
-    done(BuildResult::TimedOut, ex);
+    done(BuildResult::TimedOut, {}, ex);
 }
 
 
@@ -182,7 +182,7 @@ void DerivationGoal::loadDerivation()
     trace("loading derivation");
 
     if (nrFailed != 0) {
-        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
+        done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
         return;
     }
 
@@ -204,10 +204,33 @@ void DerivationGoal::haveDerivation()
 {
     trace("have derivation");
 
-    if (drv->type() == DerivationType::CAFloating)
+    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+
+    if (!drv->type().hasKnownOutputPaths())
         settings.requireExperimentalFeature(Xp::CaDerivations);
 
-    retrySubstitution = false;
+    if (!drv->type().isPure()) {
+        settings.requireExperimentalFeature(Xp::ImpureDerivations);
+
+        for (auto & [outputName, output] : drv->outputs) {
+            auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
+            assert(!worker.store.isValidPath(randomPath));
+            initialOutputs.insert({
+                outputName,
+                InitialOutput {
+                    .wanted = true,
+                    .outputHash = impureOutputHash,
+                    .known = InitialOutputStatus {
+                        .path = randomPath,
+                        .status = PathStatus::Absent
+                    }
+                }
+            });
+        }
+
+        gaveUpOnSubstitution();
+        return;
+    }
 
     for (auto & i : drv->outputsAndOptPaths(worker.store))
         if (i.second.second)
@@ -217,32 +240,21 @@ void DerivationGoal::haveDerivation()
     for (auto & [outputName, outputHash] : outputHashes)
         initialOutputs.insert({
             outputName,
-            InitialOutput{
+            InitialOutput {
                 .wanted = true, // Will be refined later
                 .outputHash = outputHash
             }
         });
 
     /* Check what outputs paths are not already valid. */
-    checkPathValidity();
+    auto [allValid, validOutputs] = checkPathValidity();
-    bool allValid = true;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
 
     /* If they are all valid, then we're done. */
     if (allValid && buildMode == bmNormal) {
-        done(BuildResult::AlreadyValid);
+        done(BuildResult::AlreadyValid, std::move(validOutputs));
         return;
     }
 
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -276,8 +288,10 @@ void DerivationGoal::outputsSubstitutionTried()
 {
     trace("all outputs substituted (maybe)");
 
+    assert(drv->type().isPure());
+
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure,
+        done(BuildResult::TransientFailure, {},
             Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
                 worker.store.printStorePath(drvPath)));
         return;
@@ -301,23 +315,17 @@ void DerivationGoal::outputsSubstitutionTried()
         return;
     }
 
-    checkPathValidity();
+    auto [allValid, validOutputs] = checkPathValidity();
-    size_t nrInvalid = 0;
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid())
-            nrInvalid++;
-    }
 
-    if (buildMode == bmNormal && nrInvalid == 0) {
+    if (buildMode == bmNormal && allValid) {
-        done(BuildResult::Substituted);
+        done(BuildResult::Substituted, std::move(validOutputs));
         return;
     }
-    if (buildMode == bmRepair && nrInvalid == 0) {
+    if (buildMode == bmRepair && allValid) {
         repairClosure();
         return;
     }
-    if (buildMode == bmCheck && nrInvalid > 0)
+    if (buildMode == bmCheck && !allValid)
         throw Error("some outputs of '%s' are not valid, so checking is not possible",
             worker.store.printStorePath(drvPath));
 
@@ -325,18 +333,27 @@ void DerivationGoal::outputsSubstitutionTried()
     gaveUpOnSubstitution();
 }
 
+
 /* At least one of the output paths could not be
    produced using a substitute.  So we have to build instead. */
 void DerivationGoal::gaveUpOnSubstitution()
 {
-    /* Make sure checkPathValidity() from now on checks all
-       outputs. */
-    wantedOutputs.clear();
-
     /* The inputs must be built before we can build this goal. */
+    inputDrvOutputs.clear();
     if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
+        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+            /* Ensure that pure, non-fixed-output derivations don't
+               depend on impure derivations. */
+            if (drv->type().isPure() && !drv->type().isFixed()) {
+                auto inputDrv = worker.evalStore.readDerivation(i.first);
+                if (!inputDrv.type().isPure())
+                    throw Error("pure derivation '%s' depends on impure derivation '%s'",
+                        worker.store.printStorePath(drvPath),
+                        worker.store.printStorePath(i.first));
+            }
+
             addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+        }
 
     /* Copy the input sources from the eval store to the build
        store. */
@@ -364,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
 
 void DerivationGoal::repairClosure()
 {
+    assert(drv->type().isPure());
+
     /* If we're repairing, we now know that our own outputs are valid.
        Now check whether the other paths in the outputs closure are
        good.  If not, then start derivation goals for the derivations
@@ -409,7 +428,7 @@ void DerivationGoal::repairClosure()
     }
 
     if (waitees.empty()) {
-        done(BuildResult::AlreadyValid);
+        done(BuildResult::AlreadyValid, assertPathValidity());
         return;
     }
 
@@ -423,7 +442,7 @@ void DerivationGoal::closureRepaired()
     if (nrFailed > 0)
         throw Error("some paths in the output closure of derivation '%s' could not be repaired",
             worker.store.printStorePath(drvPath));
-    done(BuildResult::AlreadyValid);
+    done(BuildResult::AlreadyValid, assertPathValidity());
 }
 
 
@@ -434,13 +453,14 @@ void DerivationGoal::inputsRealised()
     if (nrFailed != 0) {
         if (!useDerivation)
             throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        done(BuildResult::DependencyFailed, Error(
+        done(BuildResult::DependencyFailed, {}, Error(
                 "%s dependencies of derivation '%s' failed to build",
                 nrFailed, worker.store.printStorePath(drvPath)));
         return;
     }
 
-    if (retrySubstitution) {
+    if (retrySubstitution && !retriedSubstitution) {
+        retriedSubstitution = true;
         haveDerivation();
         return;
     }
@@ -454,19 +474,40 @@ void DerivationGoal::inputsRealised()
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
+        auto drvType = fullDrv.type();
-            ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
+        bool resolveDrv = std::visit(overloaded {
-            || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
+            [&](const DerivationType::InputAddressed & ia) {
+                /* must resolve if deferred. */
+                return ia.deferred;
+            },
+            [&](const DerivationType::ContentAddressed & ca) {
+                return !fullDrv.inputDrvs.empty() && (
+                    ca.fixed
+                    /* Can optionally resolve if fixed, which is good
+                       for avoiding unnecessary rebuilds. */
+                    ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
+                    /* Must resolve if floating and there are any inputs
+                       drvs. */
+                    : true);
+            },
+            [&](const DerivationType::Impure &) {
+                return true;
+            }
+        }, drvType.raw());
+
+        if (resolveDrv && !fullDrv.inputDrvs.empty()) {
+            settings.requireExperimentalFeature(Xp::CaDerivations);
+
             /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
+               now-known results of dependencies. If so, we become a
-               aliasing that resolved derivation goal */
+               stub goal aliasing that resolved derivation goal. */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
+            std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
             assert(attempt);
             Derivation drvResolved { *std::move(attempt) };
 
             auto pathResolved = writeDerivation(worker.store, drvResolved);
 
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
+            auto msg = fmt("resolved derivation: '%s' -> '%s'",
                 worker.store.printStorePath(drvPath),
                 worker.store.printStorePath(pathResolved));
             act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
@@ -487,23 +528,15 @@ void DerivationGoal::inputsRealised()
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            assert(worker.evalStore.isValidPath(drvPath));
+            for (auto & j : wantedDepOutputs)
-            auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
-            for (auto & j : wantedDepOutputs) {
+                    worker.store.computeFSClosure(*outPath, inputPaths);
-                if (outputs.count(j) > 0) {
+                else
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
                     throw Error(
                         "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
                         worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
         }
     }
-    }
 
     /* Second, the input sources. */
     worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
@@ -515,7 +548,7 @@ void DerivationGoal::inputsRealised()
 
     /* Don't repeat fixed-output derivations since they're already
        verified by their output hash.*/
-    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
+    nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1;
 
     /* Okay, try to build.  Note that here we don't wait for a build
        slot to become available, since we don't need one if there is a
@@ -523,10 +556,11 @@ void DerivationGoal::inputsRealised()
     state = &DerivationGoal::tryToBuild;
     worker.wakeUp(shared_from_this());
 
-    result = BuildResult();
+    buildResult = BuildResult { .path = buildResult.path };
 }
 
-void DerivationGoal::started() {
+void DerivationGoal::started()
+{
     auto msg = fmt(
         buildMode == bmRepair ? "repairing outputs of '%s'" :
         buildMode == bmCheck ? "checking outputs of '%s'" :
@@ -588,19 +622,12 @@ void DerivationGoal::tryToBuild()
        omitted, but that would be less efficient.)  Note that since we
        now hold the locks on the output paths, no other process can
        build this derivation, so no further checks are necessary. */
-    checkPathValidity();
+    auto [allValid, validOutputs] = checkPathValidity();
-    bool allValid = true;
+
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.wanted) continue;
-        if (!status.known || !status.known->isValid()) {
-            allValid = false;
-            break;
-        }
-    }
     if (buildMode != bmCheck && allValid) {
         debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
         outputLocks.setDeletion(true);
-        done(BuildResult::AlreadyValid);
+        done(BuildResult::AlreadyValid, std::move(validOutputs));
         return;
     }
 
@@ -626,7 +653,7 @@ void DerivationGoal::tryToBuild()
                 /* Yes, it has started doing so.  Wait until we get
                    EOF from the hook. */
                 actLock.reset();
-                result.startTime = time(0); // inexact
+                buildResult.startTime = time(0); // inexact
                 state = &DerivationGoal::buildDone;
                 started();
                 return;
@@ -830,8 +857,8 @@ void DerivationGoal::buildDone()
 
     debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
 
-    result.timesBuilt++;
+    buildResult.timesBuilt++;
-    result.stopTime = time(0);
+    buildResult.stopTime = time(0);
 
     /* So the child is gone now. */
     worker.childTerminated(this);
@@ -876,11 +903,11 @@ void DerivationGoal::buildDone()
 
         /* Compute the FS closure of the outputs and register them as
            being valid. */
-        registerOutputs();
+        auto builtOutputs = registerOutputs();
 
         StorePathSet outputPaths;
-        for (auto & [_, path] : finalOutputs)
+        for (auto & [_, output] : buildResult.builtOutputs)
-            outputPaths.insert(path);
+            outputPaths.insert(output.outPath);
         runPostBuildHook(
             worker.store,
             *logger,
@@ -890,7 +917,7 @@ void DerivationGoal::buildDone()
 
         if (buildMode == bmCheck) {
             cleanupPostOutputsRegisteredModeCheck();
-            done(BuildResult::Built);
+            done(BuildResult::Built, std::move(builtOutputs));
             return;
         }
 
@@ -911,6 +938,8 @@ void DerivationGoal::buildDone()
         outputLocks.setDeletion(true);
         outputLocks.unlock();
 
+        done(BuildResult::Built, std::move(builtOutputs));
+
     } catch (BuildError & e) {
         outputLocks.unlock();
 
@@ -926,21 +955,26 @@ void DerivationGoal::buildDone()
             st =
                 dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
                 statusOk(status) ? BuildResult::OutputRejected :
-                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
+                !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
                 BuildResult::PermanentFailure;
         }
 
-        done(st, e);
+        done(st, {}, e);
         return;
     }
-
-    done(BuildResult::Built);
 }
 
-void DerivationGoal::resolvedFinished() {
+void DerivationGoal::resolvedFinished()
+{
+    trace("resolved derivation finished");
+
     assert(resolvedDrvGoal);
     auto resolvedDrv = *resolvedDrvGoal->drv;
+    auto & resolvedResult = resolvedDrvGoal->buildResult;
 
+    DrvOutputs builtOutputs;
+
+    if (resolvedResult.success()) {
         auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
 
         StorePathSet outputPaths;
@@ -953,24 +987,19 @@ void DerivationGoal::resolvedFinished() {
         for (auto & wantedOutput : realWantedOutputs) {
             assert(initialOutputs.count(wantedOutput) != 0);
             assert(resolvedHashes.count(wantedOutput) != 0);
-        auto realisation = worker.store.queryRealisation(
+            auto realisation = resolvedResult.builtOutputs.at(
-                DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
+                DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
-        );
+            if (drv->type().isPure()) {
-        // We've just built it, but maybe the build failed, in which case the
+                auto newRealisation = realisation;
-        // realisation won't be there
+                newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
-        if (realisation) {
-            auto newRealisation = *realisation;
-            newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
                 newRealisation.signatures.clear();
-            newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
+                if (!drv->type().isFixed())
+                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
                 signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
-            outputPaths.insert(realisation->outPath);
-        } else {
-            // If we don't have a realisation, then it must mean that something
-            // failed when building the resolved drv
-            assert(!result.success());
             }
+            outputPaths.insert(realisation.outPath);
+            builtOutputs.emplace(realisation.id, realisation);
         }
 
         runPostBuildHook(
@@ -979,18 +1008,13 @@ void DerivationGoal::resolvedFinished() {
             drvPath,
             outputPaths
         );
-
-    auto status = [&]() {
-        auto resolvedResult = resolvedDrvGoal->getResult();
-        switch (resolvedResult.status) {
-            case BuildResult::AlreadyValid:
-                return BuildResult::ResolvesToAlreadyValid;
-            default:
-                return resolvedResult.status;
     }
-    }();
 
-    done(status);
+    auto status = resolvedResult.status;
+    if (status == BuildResult::AlreadyValid)
+        status = BuildResult::ResolvesToAlreadyValid;
+
+    done(status, std::move(builtOutputs));
 }
 
 HookReply DerivationGoal::tryBuildHook()
@@ -1013,7 +1037,7 @@ HookReply DerivationGoal::tryBuildHook()
 
         /* Read the first line of input, which should be a word indicating
            whether the hook wishes to perform the build. */
-        string reply;
+        std::string reply;
         while (true) {
             auto s = [&]() {
                 try {
@@ -1025,8 +1049,8 @@ HookReply DerivationGoal::tryBuildHook()
             }();
             if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
                 ;
-            else if (string(s, 0, 2) == "# ") {
+            else if (s.substr(0, 2) == "# ") {
-                reply = string(s, 2);
+                reply = s.substr(2);
                 break;
             }
             else {
@@ -1091,7 +1115,7 @@ HookReply DerivationGoal::tryBuildHook()
     /* Create the log file and pipe. */
     Path logFile = openLogFile();
 
-    set<int> fds;
+    std::set<int> fds;
     fds.insert(hook->fromHook.readSide.get());
     fds.insert(hook->builderOut.readSide.get());
     worker.childStarted(shared_from_this(), fds, false, false);
@@ -1100,7 +1124,7 @@ HookReply DerivationGoal::tryBuildHook()
 }
 
 
-void DerivationGoal::registerOutputs()
+DrvOutputs DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
        as valid (by doing `nix-store --import').  If so we don't have
@@ -1109,21 +1133,7 @@ void DerivationGoal::registerOutputs()
        We can only early return when the outputs are known a priori. For
        floating content-addressed derivations this isn't the case.
      */
-    for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
+    return assertPathValidity();
-        if (!wantOutput(outputName, wantedOutputs))
-            continue;
-        if (!optOutputPath)
-            throw BuildError(
-                "output '%s' from derivation '%s' does not have a known output path",
-                outputName, worker.store.printStorePath(drvPath));
-        auto & outputPath = *optOutputPath;
-        if (!worker.store.isValidPath(outputPath))
-            throw BuildError(
-                "output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid",
-                outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath));
-
-        finalOutputs.insert_or_assign(outputName, outputPath);
-    }
 }
 
 Path DerivationGoal::openLogFile()
@@ -1140,10 +1150,10 @@ Path DerivationGoal::openLogFile()
         logDir = localStore->logDir;
     else
         logDir = settings.nixLogDir;
-    Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
+    Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
     createDirs(dir);
 
-    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
+    Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2),
         settings.compressLog ? ".bz2" : "");
 
     fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
@@ -1175,16 +1185,17 @@ bool DerivationGoal::isReadDesc(int fd)
     return fd == hook->builderOut.readSide.get();
 }
 
-
+void DerivationGoal::handleChildOutput(int fd, std::string_view data)
-void DerivationGoal::handleChildOutput(int fd, const string & data)
 {
-    if (isReadDesc(fd))
+    // local & `ssh://`-builds are dealt with here.
+    auto isWrittenToLog = isReadDesc(fd);
+    if (isWrittenToLog)
     {
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
             killChild();
             done(
-                BuildResult::LogLimitExceeded,
+                BuildResult::LogLimitExceeded, {},
                 Error("%s killed after writing more than %d bytes of log output",
                     getName(), settings.maxLogSize));
             return;
@@ -1207,7 +1218,16 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     if (hook && fd == hook->fromHook.readSide.get()) {
         for (auto c : data)
             if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
+                auto json = parseJSONMessage(currentHookLine);
+                if (json) {
+                    auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
+                    // ensure that logs from a builder using `ssh-ng://` as protocol
+                    // are also available to `nix log`.
+                    if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
+                        auto f = (*json)["fields"];
+                        (*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
+                    }
+                }
                 currentHookLine.clear();
             } else
                 currentHookLine += c;
@@ -1241,7 +1261,8 @@ void DerivationGoal::flushLine()
 
 std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
 {
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+    assert(drv->type().isPure());
+    if (!useDerivation || drv->type().hasKnownOutputPaths()) {
         std::map<std::string, std::optional<StorePath>> res;
         for (auto & [name, output] : drv->outputs)
             res.insert_or_assign(name, output.path(worker.store, drv->name, name));
@@ -1253,7 +1274,8 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
 
 OutputPathMap DerivationGoal::queryDerivationOutputMap()
 {
-    if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+    assert(drv->type().isPure());
+    if (!useDerivation || drv->type().hasKnownOutputPaths()) {
         OutputPathMap res;
         for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
             res.insert_or_assign(name, *output.second);
@@ -1264,10 +1286,14 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
 }
 
 
-void DerivationGoal::checkPathValidity()
+std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
 {
+    if (!drv->type().isPure()) return { false, {} };
+
     bool checkHash = buildMode == bmRepair;
     auto wantedOutputsLeft = wantedOutputs;
+    DrvOutputs validOutputs;
+
     for (auto & i : queryPartialDerivationOutputMap()) {
         InitialOutput & info = initialOutputs.at(i.first);
         info.wanted = wantOutput(i.first, wantedOutputs);
@@ -1284,27 +1310,30 @@ void DerivationGoal::checkPathValidity()
                     : PathStatus::Corrupt,
             };
         }
-        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
             if (auto real = worker.store.queryRealisation(drvOutput)) {
                 info.known = {
                     .path = real->outPath,
                     .status = PathStatus::Valid,
                 };
-            } else if (info.known && info.known->status == PathStatus::Valid) {
+            } else if (info.known && info.known->isValid()) {
-                // We know the output because it' a static output of the
+                // We know the output because it's a static output of the
                 // derivation, and the output path is valid, but we don't have
                 // its realisation stored (probably because it has been built
-                // without the `ca-derivations` experimental flag)
+                // without the `ca-derivations` experimental flag).
                 worker.store.registerDrvOutput(
-                    Realisation{
+                    Realisation {
                         drvOutput,
                         info.known->path,
                     }
                 );
             }
         }
+        if (info.wanted && info.known && info.known->isValid())
+            validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
     }
+
     // If we requested all the outputs via the empty set, we are always fine.
     // If we requested specific elements, the loop above removes all the valid
     // ones, so any that are left must be invalid.
@@ -1312,24 +1341,49 @@ void DerivationGoal::checkPathValidity()
         throw Error("derivation '%s' does not have wanted outputs %s",
             worker.store.printStorePath(drvPath),
             concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
+
+    bool allValid = true;
+    for (auto & [_, status] : initialOutputs) {
+        if (!status.wanted) continue;
+        if (!status.known || !status.known->isValid()) {
+            allValid = false;
+            break;
+        }
+    }
+
+    return { allValid, validOutputs };
 }
 
 
-void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
+DrvOutputs DerivationGoal::assertPathValidity()
 {
-    result.status = status;
+    auto [allValid, validOutputs] = checkPathValidity();
+    if (!allValid)
+        throw Error("some outputs are unexpectedly invalid");
+    return validOutputs;
+}
+
+
+void DerivationGoal::done(
+    BuildResult::Status status,
+    DrvOutputs builtOutputs,
+    std::optional<Error> ex)
+{
+    buildResult.status = status;
     if (ex)
-        result.errorMsg = ex->what();
+        // FIXME: strip: "error: "
-    amDone(result.success() ? ecSuccess : ecFailed, ex);
+        buildResult.errorMsg = ex->what();
-    if (result.status == BuildResult::TimedOut)
+    if (buildResult.status == BuildResult::TimedOut)
         worker.timedOut = true;
-    if (result.status == BuildResult::PermanentFailure)
+    if (buildResult.status == BuildResult::PermanentFailure)
         worker.permanentFailure = true;
 
     mcExpectedBuilds.reset();
     mcRunningBuilds.reset();
 
-    if (result.success()) {
+    if (buildResult.success()) {
+        assert(!builtOutputs.empty());
+        buildResult.builtOutputs = std::move(builtOutputs);
         if (status == BuildResult::Built)
             worker.doneBuilds++;
     } else {
@@ -1343,9 +1397,23 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
     if (traceBuiltOutputsFile != "") {
         std::fstream fs;
         fs.open(traceBuiltOutputsFile, std::fstream::out);
-        fs << worker.store.printStorePath(drvPath) << "\t" << result.toString() << std::endl;
+        fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
     }
+
+    amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
 }
 
 
+void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
+{
+    Goal::waiteeDone(waitee, result);
+
+    if (waitee->buildResult.success())
+        if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
+            for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
+                inputDrvOutputs.insert_or_assign(
+                    { bfd->drvPath, output.outputName },
+                    realisation.outPath);
+}
+
 }

src/libstore/build/derivation-goal.hh

@@ -57,12 +57,21 @@ struct DerivationGoal : public Goal
        them. */
     StringSet wantedOutputs;
 
+    /* Mapping from input derivations + output names to actual store
+       paths. This is filled in by waiteeDone() as each dependency
+       finishes, before inputsRealised() is reached, */
+    std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
+
     /* Whether additional wanted outputs have been added. */
     bool needRestart = false;
 
     /* Whether to retry substituting the outputs after building the
-       inputs. */
+       inputs. This is done in case of an incomplete closure. */
-    bool retrySubstitution;
+    bool retrySubstitution = false;
+
+    /* Whether we've retried substitution, in which case we won't try
+       again. */
+    bool retriedSubstitution = false;
 
     /* The derivation stored at drvPath. */
     std::unique_ptr<Derivation> drv;
@@ -104,20 +113,8 @@ struct DerivationGoal : public Goal
     typedef void (DerivationGoal::*GoalState)();
     GoalState state;
 
-    /* The final output paths of the build.
-
-       - For input-addressed derivations, always the precomputed paths
-
-       - For content-addressed derivations, calcuated from whatever the hash
-         ends up being. (Note that fixed outputs derivations that produce the
-         "wrong" output still install that data under its true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
     BuildMode buildMode;
 
-    BuildResult result;
-
     /* The current round, if we're building multiple times. */
     size_t curRound = 1;
 
@@ -145,15 +142,13 @@ struct DerivationGoal : public Goal
 
     void timedOut(Error && ex) override;
 
-    string key() override;
+    std::string key() override;
 
     void work() override;
 
     /* Add wanted outputs to an already existing derivation goal. */
     void addWantedOutputs(const StringSet & outputs);
 
-    BuildResult getResult() { return result; }
-
     /* The states. */
     void getDerivation();
     void loadDerivation();
@@ -175,7 +170,7 @@ struct DerivationGoal : public Goal
 
     /* Check that the derivation outputs all exist and register them
        as valid. */
-    virtual void registerOutputs();
+    virtual DrvOutputs registerOutputs();
 
     /* Open a log file and a pipe to it. */
     Path openLogFile();
@@ -200,7 +195,7 @@ struct DerivationGoal : public Goal
     virtual bool isReadDesc(int fd);
 
     /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
+    void handleChildOutput(int fd, std::string_view data) override;
     void handleEOF(int fd) override;
     void flushLine();
 
@@ -210,8 +205,17 @@ struct DerivationGoal : public Goal
     std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
     OutputPathMap queryDerivationOutputMap();
 
-    /* Return the set of (in)valid paths. */
+    /* Update 'initialOutputs' to determine the current status of the
-    void checkPathValidity();
+       outputs of the derivation. Also returns a Boolean denoting
+       whether all outputs are valid and non-corrupt, and a
+       'DrvOutputs' structure containing the valid and wanted
+       outputs. */
+    std::pair<bool, DrvOutputs> checkPathValidity();
+
+    /* Aborts if any output is not valid or corrupt, and otherwise
+       returns a 'DrvOutputs' structure containing the wanted
+       outputs. */
+    DrvOutputs assertPathValidity();
 
     /* Forcibly kill the child process, if any. */
     virtual void killChild();
@@ -222,8 +226,11 @@ struct DerivationGoal : public Goal
 
     void done(
         BuildResult::Status status,
+        DrvOutputs builtOutputs = {},
         std::optional<Error> ex = {});
 
+    void waiteeDone(GoalPtr waitee, ExitCode result) override;
+
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 

src/libstore/build/drv-output-substitution-goal.cc

@@ -6,8 +6,12 @@
 
 namespace nix {
 
-DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
-    : Goal(worker)
+    const DrvOutput & id,
+    Worker & worker,
+    RepairFlag repair,
+    std::optional<ContentAddress> ca)
+    : Goal(worker, DerivedPath::Opaque { StorePath::dummy })
     , id(id)
 {
     state = &DrvOutputSubstitutionGoal::init;
@@ -32,12 +36,12 @@ void DrvOutputSubstitutionGoal::init()
 
 void DrvOutputSubstitutionGoal::tryNext()
 {
-    trace("Trying next substituter");
+    trace("trying next substituter");
 
     if (subs.size() == 0) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
-        debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
+        debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
 
         /* Hack: don't indicate failure if there were no substituters.
            In that case the calling derivation should just do a
@@ -119,7 +123,7 @@ void DrvOutputSubstitutionGoal::realisationFetched()
 void DrvOutputSubstitutionGoal::outPathValid()
 {
     assert(outputInfo);
-    trace("Output path substituted");
+    trace("output path substituted");
 
     if (nrFailed > 0) {
         debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
@@ -137,7 +141,7 @@ void DrvOutputSubstitutionGoal::finished()
     amDone(ecSuccess);
 }
 
-string DrvOutputSubstitutionGoal::key()
+std::string DrvOutputSubstitutionGoal::key()
 {
     /* "a$" ensures substitution goals happen before derivation
        goals. */

src/libstore/build/drv-output-substitution-goal.hh

@@ -51,7 +51,7 @@ public:
 
     void timedOut(Error && ex) override { abort(); };
 
-    string key() override;
+    std::string key() override;
 
     void work() override;
     void handleEOF(int fd) override;

src/libstore/build/entry-points.cc

@@ -47,43 +47,51 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
     }
 }
 
+std::vector<BuildResult> Store::buildPathsWithResults(
+    const std::vector<DerivedPath> & reqs,
+    BuildMode buildMode,
+    std::shared_ptr<Store> evalStore)
+{
+    Worker worker(*this, evalStore ? *evalStore : *this);
+
+    Goals goals;
+    for (const auto & br : reqs) {
+        std::visit(overloaded {
+            [&](const DerivedPath::Built & bfd) {
+                goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
+            },
+            [&](const DerivedPath::Opaque & bo) {
+                goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
+            },
+        }, br.raw());
+    }
+
+    worker.run(goals);
+
+    std::vector<BuildResult> results;
+
+    for (auto & i : goals)
+        results.push_back(i->buildResult);
+
+    return results;
+}
+
 BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
     Worker worker(*this, *this);
     auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
 
-    BuildResult result;
-
     try {
         worker.run(Goals{goal});
-        result = goal->getResult();
+        return goal->buildResult;
     } catch (Error & e) {
-        result.status = BuildResult::MiscFailure;
+        return BuildResult {
-        result.errorMsg = e.msg();
+            .status = BuildResult::MiscFailure,
-    }
+            .errorMsg = e.msg(),
-    // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
+            .path = DerivedPath::Built { .drvPath = drvPath },
-    // extended to return the full realisation for each output
+        };
-    auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
+    };
-    auto outputHashes = staticOutputHashes(*this, drv);
-    for (auto & [outputName, staticOutput] : staticDrvOutputs) {
-        auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
-        if (staticOutput.second)
-            result.builtOutputs.insert_or_assign(
-                    outputId,
-                    Realisation{ outputId, *staticOutput.second}
-                    );
-        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
-            auto realisation = this->queryRealisation(outputId);
-            if (realisation)
-                result.builtOutputs.insert_or_assign(
-                        outputId,
-                        *realisation
-                        );
-        }
-    }
-
-    return result;
 }
 
 

src/libstore/build/goal.cc

@@ -5,8 +5,8 @@ namespace nix {
 
 
 bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
-    string s1 = a->key();
+    std::string s1 = a->key();
-    string s2 = b->key();
+    std::string s2 = b->key();
     return s1 < s2;
 }
 
@@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee)
 
 void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
 {
-    assert(waitees.find(waitee) != waitees.end());
+    assert(waitees.count(waitee));
     waitees.erase(waitee);
 
     trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));

src/libstore/build/goal.hh

@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "store-api.hh"
+#include "build-result.hh"
 
 namespace nix {
 
@@ -18,8 +19,8 @@ struct CompareGoalPtrs {
 };
 
 /* Set of goals. */
-typedef set<GoalPtr, CompareGoalPtrs> Goals;
+typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
-typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
+typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
 
 /* A map of paths to goals (and the other way around). */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
@@ -39,30 +40,32 @@ struct Goal : public std::enable_shared_from_this<Goal>
     WeakGoals waiters;
 
     /* Number of goals we are/were waiting for that have failed. */
-    unsigned int nrFailed;
+    size_t nrFailed = 0;
 
     /* Number of substitution goals we are/were waiting for that
        failed because there are no substituters. */
-    unsigned int nrNoSubstituters;
+    size_t nrNoSubstituters = 0;
 
     /* Number of substitution goals we are/were waiting for that
        failed because they had unsubstitutable references. */
-    unsigned int nrIncompleteClosure;
+    size_t nrIncompleteClosure = 0;
 
     /* Name of this goal for debugging purposes. */
-    string name;
+    std::string name;
 
     /* Whether the goal is finished. */
-    ExitCode exitCode;
+    ExitCode exitCode = ecBusy;
+
+    /* Build result. */
+    BuildResult buildResult;
 
     /* Exception containing an error message, if any. */
     std::optional<Error> ex;
 
-    Goal(Worker & worker) : worker(worker)
+    Goal(Worker & worker, DerivedPath path)
-    {
+        : worker(worker)
-        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
+        , buildResult { .path = std::move(path) }
-        exitCode = ecBusy;
+    { }
-    }
 
     virtual ~Goal()
     {
@@ -75,7 +78,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
 
     virtual void waiteeDone(GoalPtr waitee, ExitCode result);
 
-    virtual void handleChildOutput(int fd, const string & data)
+    virtual void handleChildOutput(int fd, std::string_view data)
     {
         abort();
     }
@@ -87,7 +90,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
 
     void trace(const FormatOrString & fs);
 
-    string getName()
+    std::string getName()
     {
         return name;
     }
@@ -97,7 +100,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
        by the worker (important!), etc. */
     virtual void timedOut(Error && ex) = 0;
 
-    virtual string key() = 0;
+    virtual std::string key() = 0;
 
     void amDone(ExitCode result, std::optional<Error> ex = {});
 

src/libstore/build/local-derivation-goal.cc

@@ -1,4 +1,5 @@
 #include "local-derivation-goal.hh"
+#include "gc-store.hh"
 #include "hook-instance.hh"
 #include "worker.hh"
 #include "builtins.hh"
@@ -193,7 +194,7 @@ void LocalDerivationGoal::tryLocalBuild() {
         outputLocks.unlock();
         buildUser.reset();
         worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e);
+        done(BuildResult::InputRejected, {}, e);
         return;
     }
 
@@ -394,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
+            useChroot = derivationType.isSandboxed() && !noChroot;
     }
 
     auto & localStore = getLocalStore();
@@ -481,12 +482,12 @@ void LocalDerivationGoal::startBuilder()
            temporary build directory.  The text files have the format used
            by `nix-store --register-validity'.  However, the deriver
            fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph").value_or("");
+        auto s = get(drv->env, "exportReferencesGraph").value_or("");
         Strings ss = tokenizeString<Strings>(s);
         if (ss.size() % 2 != 0)
             throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
         for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-            string fileName = *i++;
+            auto fileName = *i++;
             static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
             if (!std::regex_match(fileName, regex))
                 throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
@@ -517,10 +518,10 @@ void LocalDerivationGoal::startBuilder()
                 i.pop_back();
             }
             size_t p = i.find('=');
-            if (p == string::npos)
+            if (p == std::string::npos)
                 dirsInChroot[i] = {i, optional};
             else
-                dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
+                dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
         }
         dirsInChroot[tmpDirInSandbox] = tmpDir;
 
@@ -607,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
                 "nogroup:x:65534:\n", sandboxGid()));
 
         /* Create /etc/hosts with localhost entry. */
-        if (!(derivationIsImpure(derivationType)))
+        if (derivationType.isSandboxed())
             writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
@@ -671,9 +672,10 @@ void LocalDerivationGoal::startBuilder()
         auto state = stBegin;
         auto lines = runProgram(settings.preBuildHook, false, args);
         auto lastPos = std::string::size_type{0};
-        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
+        for (auto nlPos = lines.find('\n'); nlPos != std::string::npos;
-                nlPos = lines.find('\n', lastPos)) {
+                nlPos = lines.find('\n', lastPos))
-            auto line = std::string{lines, lastPos, nlPos - lastPos};
+        {
+            auto line = lines.substr(lastPos, nlPos - lastPos);
             lastPos = nlPos + 1;
             if (state == stBegin) {
                 if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
@@ -686,10 +688,10 @@ void LocalDerivationGoal::startBuilder()
                     state = stBegin;
                 } else {
                     auto p = line.find('=');
-                    if (p == string::npos)
+                    if (p == std::string::npos)
                         dirsInChroot[line] = line;
                     else
-                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
+                        dirsInChroot[line.substr(0, p)] = line.substr(p + 1);
                 }
             }
         }
@@ -754,7 +756,7 @@ void LocalDerivationGoal::startBuilder()
     if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
         throw SysError("putting pseudoterminal into raw mode");
 
-    result.startTime = time(0);
+    buildResult.startTime = time(0);
 
     /* Fork a child to build the package. */
 
@@ -794,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
            us.
         */
 
-        if (!(derivationIsImpure(derivationType)))
+        if (derivationType.isSandboxed())
             privateNetwork = true;
 
         userNamespaceSync.create();
@@ -912,9 +914,12 @@ void LocalDerivationGoal::startBuilder()
         sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
         if (sandboxMountNamespace.get() == -1)
             throw SysError("getting sandbox mount namespace");
+
+        if (usingUserNamespace) {
             sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY);
             if (sandboxUserNamespace.get() == -1)
                 throw SysError("getting sandbox user namespace");
+        }
 
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
@@ -938,7 +943,7 @@ void LocalDerivationGoal::startBuilder()
     /* Check if setting up the build environment failed. */
     std::vector<std::string> msgs;
     while (true) {
-        string msg = [&]() {
+        std::string msg = [&]() {
             try {
                 return readLine(builderOut.readSide.get());
             } catch (Error & e) {
@@ -950,8 +955,8 @@ void LocalDerivationGoal::startBuilder()
                 throw;
             }
         }();
-        if (string(msg, 0, 1) == "\2") break;
+        if (msg.substr(0, 1) == "\2") break;
-        if (string(msg, 0, 1) == "\1") {
+        if (msg.substr(0, 1) == "\1") {
             FdSource source(builderOut.readSide.get());
             auto ex = readError(source);
             ex.addTrace({}, "while setting up the build environment");
@@ -987,7 +992,7 @@ void LocalDerivationGoal::initTmpDir() {
                 env[i.first] = i.second;
             } else {
                 auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string(Base32, false);
+                std::string fn = ".attr-" + hash.to_string(Base32, false);
                 Path p = tmpDir + "/" + fn;
                 writeFile(p, rewriteStrings(i.second, inputRewrites));
                 chownToBuilder(p);
@@ -1044,7 +1049,7 @@ void LocalDerivationGoal::initEnv()
        derivation, tell the builder, so that for instance `fetchurl'
        can skip checking the output.  On older Nixes, this environment
        variable won't be set, so `fetchurl' will do the check. */
-    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
+    if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1";
 
     /* *Only* if this is a fixed-output derivation, propagate the
        values of the environment variables specified in the
@@ -1055,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
        to the builder is generally impure, but the output of
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
-    if (derivationIsImpure(derivationType)) {
+    if (!derivationType.isSandboxed()) {
         for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
             env[i] = getEnv(i).value_or("");
     }
@@ -1078,7 +1083,7 @@ void LocalDerivationGoal::writeStructuredAttrs()
         for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
             /* The placeholder must have a rewrite, so we use it to cover both the
                cases where we know or don't know the output path ahead of time. */
-            rewritten[i] = rewriteStrings(v, inputRewrites);
+            rewritten[i] = rewriteStrings((std::string) v, inputRewrites);
         }
 
         json["outputs"] = rewritten;
@@ -1123,7 +1128,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
 /* A wrapper around LocalStore that only allows building/querying of
    paths that are in the input closures of the build or were added via
    recursive Nix calls. */
-struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
+struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
 {
     ref<LocalStore> next;
 
@@ -1184,10 +1189,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { throw Error("queryPathFromHashPart"); }
 
-    StorePath addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+        std::string_view name,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair,
+        const Path & srcPath,
-        const StorePathSet & references = StorePathSet()) override
+        FileIngestionMethod method,
+        HashType hashAlgo,
+        PathFilter & filter,
+        RepairFlag repair,
+        const StorePathSet & references) override
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
@@ -1197,17 +1206,24 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
         goal.addDependency(info.path);
     }
 
-    StorePath addTextToStore(const string & name, const string & s,
+    StorePath addTextToStore(
-        const StorePathSet & references, RepairFlag repair = NoRepair) override
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references,
+        RepairFlag repair = NoRepair) override
     {
         auto path = next->addTextToStore(name, s, references, repair);
         goal.addDependency(path);
         return path;
     }
 
-    StorePath addToStoreFromDump(Source & dump, const string & name,
+    StorePath addToStoreFromDump(
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
+        Source & dump,
-        const StorePathSet & references = StorePathSet()) override
+        std::string_view name,
+        FileIngestionMethod method,
+        HashType hashAlgo,
+        RepairFlag repair,
+        const StorePathSet & references) override
     {
         auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
         goal.addDependency(path);
@@ -1244,6 +1260,16 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     }
 
     void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
+    {
+        for (auto & result : buildPathsWithResults(paths, buildMode, evalStore))
+            if (!result.success())
+                result.rethrow();
+    }
+
+    std::vector<BuildResult> buildPathsWithResults(
+        const std::vector<DerivedPath> & paths,
+        BuildMode buildMode = bmNormal,
+        std::shared_ptr<Store> evalStore = nullptr) override
     {
         assert(!evalStore);
 
@@ -1257,25 +1283,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
                 throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
         }
 
-        next->buildPaths(paths, buildMode);
+        auto results = next->buildPathsWithResults(paths, buildMode);
 
-        for (auto & path : paths) {
+        for (auto & result : results) {
-            auto p =  std::get_if<DerivedPath::Built>(&path);
+            for (auto & [outputName, output] : result.builtOutputs) {
-            if (!p) continue;
+                newPaths.insert(output.outPath);
-            auto & bfd = *p;
+                newRealisations.insert(output);
-            auto drv = readDerivation(bfd.drvPath);
-            auto drvHashes = staticOutputHashes(*this, drv);
-            auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
-            for (auto & [outputName, outputPath] : outputs)
-                if (wantOutput(outputName, bfd.outputs)) {
-                    newPaths.insert(outputPath);
-                    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
-                        auto thisRealisation = next->queryRealisation(
-                            DrvOutput{drvHashes.at(outputName), outputName}
-                        );
-                        assert(thisRealisation);
-                        newRealisations.insert(*thisRealisation);
-                    }
             }
         }
 
@@ -1285,6 +1298,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             goal.addDependency(path);
         for (auto & real : Realisation::closure(*next, newRealisations))
             goal.addedDrvOutputs.insert(real.id);
+
+        return results;
     }
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@@ -1325,6 +1340,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
         next->queryMissing(allowed, willBuild, willSubstitute,
             unknown, downloadSize, narSize);
     }
+
+    virtual std::optional<std::string> getBuildLog(const StorePath & path) override
+    { return std::nullopt; }
+
+    virtual void addBuildLog(const StorePath & path, std::string_view log) override
+    { unsupported("addBuildLog"); }
 };
 
 
@@ -1653,7 +1674,7 @@ void LocalDerivationGoal::runChild()
             /* Fixed-output derivations typically need to access the
                network, so give them access to /etc/resolv.conf and so
                on. */
-            if (derivationIsImpure(derivationType)) {
+            if (!derivationType.isSandboxed()) {
                 // Only use nss functions to resolve hosts and
                 // services. Don’t use it for anything else that may
                 // be configured for this system. This limits the
@@ -1897,7 +1918,7 @@ void LocalDerivationGoal::runChild()
 
                 sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
 
-                if (derivationIsImpure(derivationType))
+                if (!derivationType.isSandboxed())
                     sandboxProfile += "(import \"sandbox-network.sb\")\n";
 
                 /* Add the output paths we'll use at build-time to the chroot */
@@ -1918,7 +1939,7 @@ void LocalDerivationGoal::runChild()
                             "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
                             i.first, i.second.source);
 
-                    string path = i.first;
+                    std::string path = i.first;
                     struct stat st;
                     if (lstat(path.c_str(), &st)) {
                         if (i.second.optional && errno == ENOENT)
@@ -1970,7 +1991,7 @@ void LocalDerivationGoal::runChild()
                 args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
                 if (allowLocalNetworking) {
                     args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+                    args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));
                 }
                 args.push_back(drv->builder);
             } else {
@@ -1989,7 +2010,7 @@ void LocalDerivationGoal::runChild()
             args.push_back(rewriteStrings(i, inputRewrites));
 
         /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, string("\2\n"));
+        writeFull(STDERR_FILENO, std::string("\2\n"));
 
         /* Execute the program.  This should not return. */
         if (drv->isBuiltin()) {
@@ -2007,7 +2028,7 @@ void LocalDerivationGoal::runChild()
                 else if (drv->builder == "builtin:unpack-channel")
                     builtinUnpackChannel(drv2);
                 else
-                    throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8));
+                    throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8));
                 _exit(0);
             } catch (std::exception & e) {
                 writeFull(STDERR_FILENO, e.what() + std::string("\n"));
@@ -2053,7 +2074,7 @@ void LocalDerivationGoal::runChild()
 }
 
 
-void LocalDerivationGoal::registerOutputs()
+DrvOutputs LocalDerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
        as valid (by doing `nix-store --import').  If so we don't have
@@ -2062,10 +2083,8 @@ void LocalDerivationGoal::registerOutputs()
        We can only early return when the outputs are known a priori. For
        floating content-addressed derivations this isn't the case.
      */
-    if (hook) {
+    if (hook)
-        DerivationGoal::registerOutputs();
+        return DerivationGoal::registerOutputs();
-        return;
-    }
 
     std::map<std::string, ValidPathInfo> infos;
 
@@ -2188,6 +2207,8 @@ void LocalDerivationGoal::registerOutputs()
 
     std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
 
+    OutputPathMap finalOutputs;
+
     for (auto & outputName : sortedOutputNames) {
         auto output = drv->outputs.at(outputName);
         auto & scratchPath = scratchOutputs.at(outputName);
@@ -2258,7 +2279,7 @@ void LocalDerivationGoal::registerOutputs()
             return res;
         };
 
-        auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
+        auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
             auto & st = outputStats.at(outputName);
             if (outputHash.method == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
@@ -2324,7 +2345,8 @@ void LocalDerivationGoal::registerOutputs()
         };
 
         ValidPathInfo newInfo = std::visit(overloaded {
-            [&](const DerivationOutputInputAddressed & output) {
+
+            [&](const DerivationOutput::InputAddressed & output) {
                 /* input-addressed case */
                 auto requiredFinalPath = output.path;
                 /* Preemptively add rewrite rule for final hash, as that is
@@ -2343,8 +2365,9 @@ void LocalDerivationGoal::registerOutputs()
                     newInfo0.references.insert(newInfo0.path);
                 return newInfo0;
             },
-            [&](const DerivationOutputCAFixed & dof) {
+
-                auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
+            [&](const DerivationOutput::CAFixed & dof) {
+                auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
                     .method = dof.hash.method,
                     .hashType = dof.hash.hash.type,
                 });
@@ -2365,19 +2388,25 @@ void LocalDerivationGoal::registerOutputs()
                 }
                 return newInfo0;
             },
-            [&](DerivationOutputCAFloating dof) {
+
+            [&](const DerivationOutput::CAFloating & dof) {
                 return newInfoFromCA(dof);
             },
-                [&](DerivationOutputDeferred) {
+
+            [&](const DerivationOutput::Deferred &) -> ValidPathInfo {
                 // No derivation should reach that point without having been
                 // rewritten first
                 assert(false);
-                // Ugly, but the compiler insists on having this return a value
-                // of type `ValidPathInfo` despite the `assert(false)`, so
-                // let's provide it
-                return *(ValidPathInfo*)0;
             },
-        }, output.output);
+
+            [&](const DerivationOutput::Impure & doi) {
+                return newInfoFromCA(DerivationOutput::CAFloating {
+                    .method = doi.method,
+                    .hashType = doi.hashType,
+                });
+            },
+
+        }, output.raw());
 
         /* FIXME: set proper permissions in restorePath() so
             we don't have to do another traversal. */
@@ -2487,11 +2516,12 @@ void LocalDerivationGoal::registerOutputs()
     }
 
     if (buildMode == bmCheck) {
-        // In case of FOD mismatches on `--check` an error must be thrown as this is also
+        /* In case of fixed-output derivations, if there are
-        // a source for non-determinism.
+           mismatches on `--check` an error must be thrown as this is
+           also a source for non-determinism. */
         if (delayedException)
             std::rethrow_exception(delayedException);
-        return;
+        return assertPathValidity();
     }
 
     /* Apply output checks. */
@@ -2503,7 +2533,7 @@ void LocalDerivationGoal::registerOutputs()
         assert(prevInfos.size() == infos.size());
         for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
             if (!(*i == *j)) {
-                result.isNonDeterministic = true;
+                buildResult.isNonDeterministic = true;
                 Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
                 bool prevExists = keepPreviousRound && pathExists(prev);
                 hintformat hint = prevExists
@@ -2541,7 +2571,7 @@ void LocalDerivationGoal::registerOutputs()
 
     if (curRound < nrRounds) {
         prevInfos = std::move(infos);
-        return;
+        return {};
     }
 
     /* Remove the .check directories if we're done. FIXME: keep them
@@ -2576,17 +2606,27 @@ void LocalDerivationGoal::registerOutputs()
        means it's safe to link the derivation to the output hash. We must do
        that for floating CA derivations, which otherwise couldn't be cached,
        but it's fine to do in all cases. */
+    DrvOutputs builtOutputs;
 
-    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+    for (auto & [outputName, newInfo] : infos) {
-        for (auto& [outputName, newInfo] : infos) {
+        auto thisRealisation = Realisation {
-            auto thisRealisation = Realisation{
+            .id = DrvOutput {
-                .id = DrvOutput{initialOutputs.at(outputName).outputHash,
+                initialOutputs.at(outputName).outputHash,
-                                outputName},
+                outputName
-                .outPath = newInfo.path};
+            },
+            .outPath = newInfo.path
+        };
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
+            && drv->type().isPure())
+        {
             signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);
         }
+        if (wantOutput(outputName, wantedOutputs))
+            builtOutputs.emplace(thisRealisation.id, thisRealisation);
     }
+
+    return builtOutputs;
 }
 
 void LocalDerivationGoal::signRealisation(Realisation & realisation)
@@ -2595,7 +2635,7 @@ void LocalDerivationGoal::signRealisation(Realisation & realisation)
 }
 
 
-void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
+void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & outputs)
 {
     std::map<Path, const ValidPathInfo &> outputsByPath;
     for (auto & output : outputs)
@@ -2667,8 +2707,8 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
                 for (auto & i : *value) {
                     if (worker.store.isStorePath(i))
                         spec.insert(worker.store.parseStorePath(i));
-                    else if (finalOutputs.count(i))
+                    else if (outputs.count(i))
-                        spec.insert(finalOutputs.at(i));
+                        spec.insert(outputs.at(i).path);
                     else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
                 }
 
@@ -2691,7 +2731,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
                     }
 
                 if (!badPaths.empty()) {
-                    string badPathsStr;
+                    std::string badPathsStr;
                     for (auto & i : badPaths) {
                         badPathsStr += "\n  ";
                         badPathsStr += worker.store.printStorePath(i);

src/libstore/build/local-derivation-goal.hh

@@ -58,11 +58,11 @@ struct LocalDerivationGoal : public DerivationGoal
     typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
     DirsInChroot dirsInChroot;
 
-    typedef map<string, string> Environment;
+    typedef map<std::string, std::string> Environment;
     Environment env;
 
 #if __APPLE__
-    typedef string SandboxProfile;
+    typedef std::string SandboxProfile;
     SandboxProfile additionalSandboxProfile;
 #endif
 
@@ -169,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal
 
     /* Check that the derivation outputs all exist and register them
        as valid. */
-    void registerOutputs() override;
+    DrvOutputs registerOutputs() override;
 
     void signRealisation(Realisation &) override;
 

src/libstore/build/substitution-goal.cc

@@ -6,7 +6,7 @@
 namespace nix {
 
 PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker)
+    : Goal(worker, DerivedPath::Opaque { storePath })
     , storePath(storePath)
     , repair(repair)
     , ca(ca)
@@ -24,6 +24,20 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
 }
 
 
+void PathSubstitutionGoal::done(
+    ExitCode result,
+    BuildResult::Status status,
+    std::optional<std::string> errorMsg)
+{
+    buildResult.status = status;
+    if (errorMsg) {
+        debug(*errorMsg);
+        buildResult.errorMsg = *errorMsg;
+    }
+    amDone(result);
+}
+
+
 void PathSubstitutionGoal::work()
 {
     (this->*state)();
@@ -38,7 +52,7 @@ void PathSubstitutionGoal::init()
 
     /* If the path already exists we're done. */
     if (!repair && worker.store.isValidPath(storePath)) {
-        amDone(ecSuccess);
+        done(ecSuccess, BuildResult::AlreadyValid);
         return;
     }
 
@@ -60,12 +74,14 @@ void PathSubstitutionGoal::tryNext()
     if (subs.size() == 0) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
-        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
 
         /* Hack: don't indicate failure if there were no substituters.
            In that case the calling derivation should just do a
            build. */
-        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
+        done(
+            substituterFailed ? ecFailed : ecNoSubstituters,
+            BuildResult::NoSubstituters,
+            fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)));
 
         if (substituterFailed) {
             worker.failedSubstitutions++;
@@ -162,8 +178,10 @@ void PathSubstitutionGoal::referencesValid()
     trace("all references realised");
 
     if (nrFailed > 0) {
-        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
+        done(
-        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
+            nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
+            BuildResult::DependencyFailed,
+            fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
         return;
     }
 
@@ -268,11 +286,11 @@ void PathSubstitutionGoal::finished()
 
     worker.updateProgress();
 
-    amDone(ecSuccess);
+    done(ecSuccess, BuildResult::Substituted);
 }
 
 
-void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
+void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data)
 {
 }
 

src/libstore/build/substitution-goal.hh

@@ -53,13 +53,18 @@ struct PathSubstitutionGoal : public Goal
     /* Content address for recomputing store path */
     std::optional<ContentAddress> ca;
 
+    void done(
+        ExitCode result,
+        BuildResult::Status status,
+        std::optional<std::string> errorMsg = {});
+
 public:
     PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
     ~PathSubstitutionGoal();
 
     void timedOut(Error && ex) override { abort(); };
 
-    string key() override
+    std::string key() override
     {
         /* "a$" ensures substitution goals happen before derivation
            goals. */
@@ -77,7 +82,7 @@ public:
     void finished();
 
     /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data) override;
+    void handleChildOutput(int fd, std::string_view data) override;
     void handleEOF(int fd) override;
 
     void cleanup() override;

src/libstore/build/worker.cc

@@ -161,7 +161,7 @@ unsigned Worker::getNrLocalBuilds()
 }
 
 
-void Worker::childStarted(GoalPtr goal, const set<int> & fds,
+void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     bool inBuildSlot, bool respectTimeouts)
 {
     Child child;
@@ -377,7 +377,7 @@ void Worker::waitForInput()
         GoalPtr goal = j->goal.lock();
         assert(goal);
 
-        set<int> fds2(j->fds);
+        std::set<int> fds2(j->fds);
         std::vector<unsigned char> buffer(4096);
         for (auto & k : fds2) {
             if (pollStatus.at(fdToPollStatus.at(k)).revents) {
@@ -394,7 +394,7 @@ void Worker::waitForInput()
                 } else {
                     printMsg(lvlVomit, "%1%: read %2% bytes",
                         goal->getName(), rd);
-                    string data((char *) buffer.data(), rd);
+                    std::string data((char *) buffer.data(), rd);
                     j->lastOutput = after;
                     goal->handleChildOutput(k, data);
                 }

src/libstore/build/worker.hh

@@ -38,7 +38,7 @@ struct Child
 {
     WeakGoalPtr goal;
     Goal * goal2; // ugly hackery
-    set<int> fds;
+    std::set<int> fds;
     bool respectTimeouts;
     bool inBuildSlot;
     steady_time_point lastOutput; /* time we last got output on stdout/stderr */
@@ -167,7 +167,7 @@ public:
 
     /* Registers a running child process.  `inBuildSlot' means that
        the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, const set<int> & fds,
+    void childStarted(GoalPtr goal, const std::set<int> & fds,
         bool inBuildSlot, bool respectTimeouts);
 
     /* Unregisters a running child process.  `wakeSleepers' should be

src/libstore/builtins/buildenv.cc

@@ -47,9 +47,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
             throw;
         }
 
-        /* The files below are special-cased to that they don't show up
+        /* The files below are special-cased to that they don't show
-         * in user profiles, either because they are useless, or
+         * up in user profiles, either because they are useless, or
-         * because they would cauase pointless collisions (e.g., each
+         * because they would cause pointless collisions (e.g., each
          * Python package brings its own
          * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
          */
@@ -57,7 +57,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
             hasSuffix(srcFile, "/nix-support") ||
             hasSuffix(srcFile, "/perllocal.pod") ||
             hasSuffix(srcFile, "/info/dir") ||
-            hasSuffix(srcFile, "/log"))
+            hasSuffix(srcFile, "/log") ||
+            hasSuffix(srcFile, "/manifest.nix") ||
+            hasSuffix(srcFile, "/manifest.json"))
             continue;
 
         else if (S_ISDIR(srcSt.st_mode)) {
@@ -123,7 +125,7 @@ void buildProfile(const Path & out, Packages && pkgs)
         createLinks(state, pkgDir, out, priority);
 
         try {
-            for (const auto & p : tokenizeString<std::vector<string>>(
+            for (const auto & p : tokenizeString<std::vector<std::string>>(
                     readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
                 if (!done.count(p))
                     postponed.insert(p);
@@ -161,7 +163,7 @@ void buildProfile(const Path & out, Packages && pkgs)
 
 void builtinBuildenv(const BasicDerivation & drv)
 {
-    auto getAttr = [&](const string & name) {
+    auto getAttr = [&](const std::string & name) {
         auto i = drv.env.find(name);
         if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;

src/libstore/builtins/fetchurl.cc

@@ -16,7 +16,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         writeFile(settings.netrcFile, netrcData, 0600);
     }
 
-    auto getAttr = [&](const string & name) {
+    auto getAttr = [&](const std::string & name) {
         auto i = drv.env.find(name);
         if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;

src/libstore/builtins/unpack-channel.cc

@@ -5,7 +5,7 @@ namespace nix {
 
 void builtinUnpackChannel(const BasicDerivation & drv)
 {
-    auto getAttr = [&](const string & name) {
+    auto getAttr = [&](const std::string & name) {
         auto i = drv.env.find(name);
         if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;

src/libstore/daemon.cc

@@ -1,7 +1,11 @@
 #include "daemon.hh"
 #include "monitor-fd.hh"
 #include "worker-protocol.hh"
+#include "build-result.hh"
 #include "store-api.hh"
+#include "store-cast.hh"
+#include "gc-store.hh"
+#include "log-store.hh"
 #include "path-with-outputs.hh"
 #include "finally.hh"
 #include "archive.hh"
@@ -479,8 +483,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddTextToStore: {
-        string suffix = readString(from);
+        std::string suffix = readString(from);
-        string s = readString(from);
+        std::string s = readString(from);
         auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
@@ -530,6 +534,25 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopBuildPathsWithResults: {
+        auto drvs = readDerivedPaths(*store, clientVersion, from);
+        BuildMode mode = bmNormal;
+        mode = (BuildMode) readInt(from);
+
+        /* Repairing is not atomic, so disallowed for "untrusted"
+           clients.  */
+        if (mode == bmRepair && !trusted)
+            throw Error("repairing is not allowed because you are not in 'trusted-users'");
+
+        logger->startWork();
+        auto results = store->buildPathsWithResults(drvs, mode);
+        logger->stopWork();
+
+        worker_proto::write(*store, to, results);
+
+        break;
+    }
+
     case wopBuildDerivation: {
         auto drvPath = store->parseStorePath(readString(from));
         BasicDerivation drv;
@@ -537,6 +560,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         BuildMode buildMode = (BuildMode) readInt(from);
         logger->startWork();
 
+        auto drvType = drv.type();
+
         /* Content-addressed derivations are trustless because their output paths
            are verified by their content alone, so any derivation is free to
            try to produce such a path.
@@ -569,12 +594,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
            derivations, we throw out the precomputed output paths and just
            store the hashes, so there aren't two competing sources of truth an
            attacker could exploit. */
-        if (drv.type() == DerivationType::InputAddressed && !trusted)
+        if (!(drvType.isCA() || trusted))
             throw Error("you are not privileged to build input-addressed derivations");
 
         /* Make sure that the non-input-addressed derivations that got this far
            are in fact content-addressed if we don't trust them. */
-        assert(derivationIsCA(drv.type()) || trusted);
+        assert(drvType.isCA() || trusted);
 
         /* Recompute the derivation path when we cannot trust the original. */
         if (!trusted) {
@@ -583,7 +608,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                original not-necessarily-resolved derivation to verify the drv
                derivation as adequate claim to the input-addressed output
                paths. */
-            assert(derivationIsCA(drv.type()));
+            assert(drvType.isCA());
 
             Derivation drv2;
             static_cast<BasicDerivation &>(drv2) = drv;
@@ -622,9 +647,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopAddIndirectRoot: {
         Path path = absPath(readString(from));
+
         logger->startWork();
-        store->addIndirectRoot(path);
+        auto & gcStore = require<GcStore>(*store);
+        gcStore.addIndirectRoot(path);
         logger->stopWork();
+
         to << 1;
         break;
     }
@@ -639,7 +667,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopFindRoots: {
         logger->startWork();
-        Roots roots = store->findRoots(!trusted);
+        auto & gcStore = require<GcStore>(*store);
+        Roots roots = gcStore.findRoots(!trusted);
         logger->stopWork();
 
         size_t size = 0;
@@ -670,7 +699,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         if (options.ignoreLiveness)
             throw Error("you are not allowed to ignore liveness");
-        store->collectGarbage(options, results);
+        auto & gcStore = require<GcStore>(*store);
+        gcStore.collectGarbage(options, results);
         logger->stopWork();
 
         to << results.paths << results.bytesFreed << 0 /* obsolete */;
@@ -698,8 +728,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
             unsigned int n = readInt(from);
             for (unsigned int i = 0; i < n; i++) {
-                string name = readString(from);
+                auto name = readString(from);
-                string value = readString(from);
+                auto value = readString(from);
                 clientSettings.overrides.emplace(name, value);
             }
         }
@@ -927,11 +957,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         if (!trusted)
             throw Error("you are not privileged to add logs");
+        auto & logStore = require<LogStore>(*store);
         {
             FramedSource source(from);
             StringSink sink;
             source.drainInto(sink);
-            store->addBuildLog(path, sink.s);
+            logStore.addBuildLog(path, sink.s);
         }
         logger->stopWork();
         to << 1;