Merge remote-tracking branch 'upstream/master' into ca-drv-exotic

.gitignore

@@ -40,6 +40,7 @@ perl/Makefile.config
 
 # /src/libstore/
 *.gen.*
+/src/libstore/tests/libstore-tests
 
 # /src/libutil/
 /src/libutil/tests/libutil-tests

.version

@@ -1 +1 @@
-2.4
+2.5

Makefile

@@ -4,6 +4,7 @@ makefiles = \
   src/libutil/local.mk \
   src/libutil/tests/local.mk \
   src/libstore/local.mk \
+  src/libstore/tests/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \

doc/manual/generate-manpage.nix

@@ -1,4 +1,4 @@
-command:
+{ command, renderLinks ? false }:
 
 with builtins;
 with import ./utils.nix;
@@ -20,7 +20,11 @@ let
            categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
            listCommands = cmds:
              concatStrings (map (name:
-               "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n")
+               "* "
+               + (if renderLinks
+                  then "[`${command} ${name}`](./${appendName filename name}.md)"
+                  else "`${command} ${name}`")
+               + " - ${cmds.${name}.description}\n")
                (attrNames cmds));
          in
          "where *subcommand* is one of the following:\n\n"

doc/manual/local.mk

@@ -44,7 +44,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@
-	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
+	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }'
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp

doc/manual/src/installation/env-variables.md

@@ -40,7 +40,7 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
 > **Note**
 > 
 > You must not add the export and then do the install, as the Nix
-> installer will detect the presense of Nix configuration, and abort.
+> installer will detect the presence of Nix configuration, and abort.
 
 ## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon
 

doc/manual/src/release-notes/rl-2.5.md

@@ -0,0 +1,2 @@
+# Release 2.5 (2021-XX-XX)
+

flake.lock

@@ -3,11 +3,11 @@
     "lowdown-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1632468475,
+        "lastModified": 1633514407,
-        "narHash": "sha256-NNOm9CbdA8cuwbvaBHslGbPTiU6bh1Ao+MpEPx4rSGo=",
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
         "owner": "kristapsdz",
         "repo": "lowdown",
-        "rev": "6bd668af3fd098bdd07a1bedd399564141e275da",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
         "type": "github"
       },
       "original": {

flake.nix

@@ -78,7 +78,7 @@
 
             # Tests
             buildPackages.git
-            buildPackages.mercurial
+            buildPackages.mercurial # FIXME: remove? only needed for tests
             buildPackages.jq
           ]
           ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
@@ -126,8 +126,7 @@
           ''
             mkdir -p $out/nix-support
 
-            # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
+            # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
-            # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
             tarballPath() {
               # Remove the store prefix
               local path=''${1#${builtins.storeDir}/}
@@ -153,13 +152,15 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
-      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
+      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
-        # Must keep this name short as OSX has a rather strict limit on the
+        name =
-        # socket path length, and this name appears in the path of the
+          "nix-tests"
-        # nix-daemon socket used in the tests
+          + optionalString
-        name = "nix-tests";
+            (versionAtLeast daemon.version "2.4pre20211005" &&
+             versionAtLeast client.version "2.4pre20211005")
+            "-${client.version}-against-${daemon.version}";
         inherit version;
 
         src = self;
@@ -443,6 +444,12 @@
           inherit (self) overlay;
         };
 
+        tests.nssPreload = (import ./tests/nss-preload.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
         tests.githubFlakes = (import ./tests/github-flakes.nix rec {
           system = "x86_64-linux";
           inherit nixpkgs;
@@ -490,7 +497,11 @@
           let pkgs = nixpkgsFor.${system}; in
           pkgs.runCommand "install-tests" {
             againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
-            againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
+            againstCurrentUnstable =
+              # FIXME: temporarily disable this on macOS because of #3605.
+              if system == "x86_64-linux"
+              then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
+              else null;
             # Disabled because the latest stable version doesn't handle
             # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
             # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;

maintainers/upload-release.pl

@@ -12,6 +12,7 @@ use LWP::UserAgent;
 use Net::Amazon::S3;
 
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
+my $nixRev = $ARGV[1] or die; # FIXME
 
 my $releasesBucketName = "nix-releases";
 my $channelsBucketName = "nix-channels";
@@ -19,6 +20,8 @@ my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
 my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
 
+my $isLatest = ($ENV{'IS_LATEST'} // "") eq "1";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -33,14 +36,15 @@ sub fetch {
 }
 
 my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
-my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
+#my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
 #print Dumper($evalInfo);
 
-my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die;
+#my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die;
 
-my $tarballInfo = decode_json(fetch("$evalUrl/job/tarball", 'application/json'));
+my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
+#print Dumper($buildInfo);
 
-my $releaseName = $tarballInfo->{releasename};
+my $releaseName = $buildInfo->{nixname};
 $releaseName =~ /nix-(.*)$/ or die;
 my $version = $1;
 
@@ -104,8 +108,6 @@ sub downloadFile {
     return $sha256_expected;
 }
 
-downloadFile("tarball", "2"); # .tar.bz2
-my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
@@ -134,42 +136,43 @@ for my $fn (glob "$tmpDir/*") {
     }
 }
 
-exit if $version =~ /pre/;
-
 # Update nix-fallback-paths.nix.
-system("cd $nixpkgsDir && git pull") == 0 or die;
+if ($isLatest) {
+    system("cd $nixpkgsDir && git pull") == 0 or die;
 
-sub getStorePath {
+    sub getStorePath {
-    my ($jobName) = @_;
+        my ($jobName) = @_;
-    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+        my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    for my $product (values %{$buildInfo->{buildproducts}}) {
+        for my $product (values %{$buildInfo->{buildproducts}}) {
-        next unless $product->{type} eq "nix-build";
+            next unless $product->{type} eq "nix-build";
-        next if $product->{path} =~ /[a-z]+$/;
+            next if $product->{path} =~ /[a-z]+$/;
-        return $product->{path};
+            return $product->{path};
+        }
+        die;
     }
-    die;
+
+    write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
+               "{\n" .
+               "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
+               "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
+               "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+               "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+               "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
+               "}\n");
+
+    system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
 }
 
-write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
-           "{\n" .
-           "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
-           "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
-           "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
-           "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
-           "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
-           "}\n");
-
-system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
-
 # Update the "latest" symlink.
 $channelsBucket->add_key(
     "nix-latest/install", "",
     { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
-    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr
+    if $isLatest;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
 system("git push --tags") == 0 or die;
-system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;

mk/libraries.mk

@@ -91,7 +91,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT)
 
     $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED)
+	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
     ifndef HOST_DARWIN
       $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
@@ -105,7 +105,7 @@ define build-library
     $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
     $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ -shared $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED)
+	$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
     ifndef HOST_DARWIN

mk/programs.mk

@@ -32,7 +32,7 @@ define build-program
   $$(eval $$(call create-dir, $$(_d)))
 
   $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$@ $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS)
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
 
@@ -49,7 +49,7 @@ define build-program
       _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
 
       $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS)
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     else
 

mk/tests.mk

@@ -13,3 +13,7 @@ define run-install-test
 endef
 
 .PHONY: check installcheck
+
+print-top-help += \
+  echo "  check: Run unit tests"; \
+  echo "  installcheck: Run functional tests";

src/libexpr/eval.cc

@@ -445,12 +445,12 @@ EvalState::EvalState(
                     StorePathSet closure;
                     store->computeFSClosure(store->toStorePath(r.second).first, closure);
                     for (auto & path : closure)
-                        allowedPaths->insert(store->printStorePath(path));
+                        allowPath(path);
                 } catch (InvalidPath &) {
-                    allowedPaths->insert(r.second);
+                    allowPath(r.second);
                 }
             } else
-                allowedPaths->insert(r.second);
+                allowPath(r.second);
         }
     }
 
@@ -482,6 +482,18 @@ void EvalState::requireExperimentalFeatureOnEvaluation(
     }
 }
 
+void EvalState::allowPath(const Path & path)
+{
+    if (allowedPaths)
+        allowedPaths->insert(path);
+}
+
+void EvalState::allowPath(const StorePath & storePath)
+{
+    if (allowedPaths)
+        allowedPaths->insert(store->toRealPath(storePath));
+}
+
 Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
@@ -1316,13 +1328,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     auto size =
         (lambda.arg.empty() ? 0 : 1) +
-        (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
+        (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
     Env & env2(allocEnv(size));
     env2.up = fun.lambda.env;
 
     size_t displ = 0;
 
-    if (!lambda.matchAttrs)
+    if (!lambda.hasFormals())
         env2.values[displ++] = &arg;
 
     else {
@@ -1402,7 +1414,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         }
     }
 
-    if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
+    if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) {
         res = fun;
         return;
     }
@@ -1889,6 +1901,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
             : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
+        allowPath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
     }

src/libexpr/eval.hh

@@ -150,6 +150,15 @@ public:
 
     SearchPath getSearchPath() { return searchPath; }
 
+    /* Allow access to a path. */
+    void allowPath(const Path & path);
+
+    /* Allow access to a store path. Note that this gets remapped to
+       the real store path if `store` is a chroot store. */
+    void allowPath(const StorePath & storePath);
+
+    /* Check whether access to a path is allowed and throw an error if
+       not. Otherwise return the canonicalised path. */
     Path checkSourcePath(const Path & path);
 
     void checkURI(const std::string & uri);

src/libexpr/flake/flake.cc

@@ -64,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     debug("got tree '%s' from '%s'",
         state.store->printStorePath(tree.storePath), lockedRef);
 
-    if (state.allowedPaths)
+    state.allowPath(tree.storePath);
-        state.allowedPaths->insert(tree.actualPath);
 
     assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
 
@@ -231,7 +230,7 @@ static Flake getFlake(
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
         expectType(state, nFunction, *outputs->value, *outputs->pos);
 
-        if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
+        if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) {
             for (auto & formal : outputs->value->lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {

src/libexpr/nixexpr.cc

@@ -124,7 +124,7 @@ void ExprList::show(std::ostream & str) const
 void ExprLambda::show(std::ostream & str) const
 {
     str << "(";
-    if (matchAttrs) {
+    if (hasFormals()) {
         str << "{ ";
         bool first = true;
         for (auto & i : formals->formals) {
@@ -348,7 +348,7 @@ void ExprLambda::bindVars(const StaticEnv & env)
 
     if (!arg.empty()) newEnv.vars[arg] = displ++;
 
-    if (matchAttrs) {
+    if (hasFormals()) {
         for (auto & i : formals->formals)
             newEnv.vars[i.name] = displ++;
 

src/libexpr/nixexpr.hh

@@ -233,11 +233,10 @@ struct ExprLambda : Expr
     Pos pos;
     Symbol name;
     Symbol arg;
-    bool matchAttrs;
     Formals * formals;
     Expr * body;
-    ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
+    ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body)
-        : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
+        : pos(pos), arg(arg), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
             throw ParseError({
@@ -247,6 +246,7 @@ struct ExprLambda : Expr
     };
     void setName(Symbol & name);
     string showNamePos() const;
+    inline bool hasFormals() const { return formals != nullptr; }
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -324,13 +324,13 @@ expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); }
   | '{' formals '}' '@' ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); }
   | ID '@' '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function

src/libexpr/primops.cc

@@ -56,13 +56,9 @@ void EvalState::realiseContext(const PathSet & context)
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             store->printStorePath(drvs.begin()->drvPath));
 
-    /* For performance, prefetch all substitute info. */
+    /* Build/substitute the context. */
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
     store->buildPaths(buildReqs);
 
     /* Add the output of this derivations to the allowed
@@ -1851,56 +1847,87 @@ static RegisterPrimOp primop_toFile({
     .fun = prim_toFile,
 });
 
-static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
+static void addPath(
-    Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
+    EvalState & state,
+    const Pos & pos,
+    const string & name,
+    Path path,
+    Value * filterFun,
+    FileIngestionMethod method,
+    const std::optional<Hash> expectedHash,
+    Value & v,
+    const PathSet & context)
 {
-    const auto path = evalSettings.pureEval && expectedHash ?
+    try {
-        path_ :
+        // FIXME: handle CA derivation outputs (where path needs to
-        state.checkSourcePath(path_);
+        // be rewritten to the actual output).
-    PathFilter filter = filterFun ? ([&](const Path & path) {
+        state.realiseContext(context);
-        auto st = lstat(path);
 
-        /* Call the filter function.  The first argument is the path,
+        if (state.store->isInStore(path)) {
-           the second is a string indicating the type of the file. */
+            auto [storePath, subPath] = state.store->toStorePath(path);
-        Value arg1;
+            auto info = state.store->queryPathInfo(storePath);
-        mkString(arg1, path);
+            if (!info->references.empty())
+                throw EvalError("store path '%s' is not allowed to have references",
+                    state.store->printStorePath(storePath));
+            path = state.store->toRealPath(storePath) + subPath;
+        }
 
-        Value fun2;
+        path = evalSettings.pureEval && expectedHash
-        state.callFunction(*filterFun, arg1, fun2, noPos);
+            ? path
+            : state.checkSourcePath(path);
 
-        Value arg2;
+        PathFilter filter = filterFun ? ([&](const Path & path) {
-        mkString(arg2,
+            auto st = lstat(path);
-            S_ISREG(st.st_mode) ? "regular" :
-            S_ISDIR(st.st_mode) ? "directory" :
-            S_ISLNK(st.st_mode) ? "symlink" :
-            "unknown" /* not supported, will fail! */);
 
-        Value res;
+            /* Call the filter function.  The first argument is the path,
-        state.callFunction(fun2, arg2, res, noPos);
+               the second is a string indicating the type of the file. */
+            Value arg1;
+            mkString(arg1, path);
 
-        return state.forceBool(res, pos);
+            Value fun2;
-    }) : defaultPathFilter;
+            state.callFunction(*filterFun, arg1, fun2, noPos);
 
-    std::optional<StorePath> expectedStorePath;
+            Value arg2;
-    if (expectedHash)
+            mkString(arg2,
-        expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
+                S_ISREG(st.st_mode) ? "regular" :
-            {
+                S_ISDIR(st.st_mode) ? "directory" :
-                .method = method,
+                S_ISLNK(st.st_mode) ? "symlink" :
-                .hash = *expectedHash,
+                "unknown" /* not supported, will fail! */);
-            },
-            {},
-        });
-    Path dstPath;
-    if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-        dstPath = state.store->printStorePath(settings.readOnlyMode
-            ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
-            : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
-        if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
-            throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
-    } else
-        dstPath = state.store->printStorePath(*expectedStorePath);
 
-    mkString(v, dstPath, {dstPath});
+            Value res;
+            state.callFunction(fun2, arg2, res, noPos);
+
+            return state.forceBool(res, pos);
+        }) : defaultPathFilter;
+
+        std::optional<StorePath> expectedStorePath;
+        if (expectedHash)
+            expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
+                {
+                    .method = method,
+                    .hash = *expectedHash,
+                },
+                {},
+            });
+
+        Path dstPath;
+        if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+            dstPath = state.store->printStorePath(settings.readOnlyMode
+                ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
+            if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+                throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
+        } else
+            dstPath = state.store->printStorePath(*expectedStorePath);
+
+        mkString(v, dstPath, {dstPath});
+
+        state.allowPath(dstPath);
+
+    } catch (Error & e) {
+        e.addTrace(pos, "while adding path '%s'", path);
+        throw;
+    }
 }
 
 
@@ -1908,11 +1935,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
-    if (!context.empty())
-        throw EvalError({
-            .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-            .errPos = pos
-        });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type() != nFunction)
@@ -1923,13 +1945,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
             .errPos = pos
         });
 
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
     .name = "__filterSource",
     .args = {"e1", "e2"},
     .doc = R"(
+      > **Warning**
+      >
+      > `filterSource` should not be used to filter store paths. Since
+      > `filterSource` uses the name of the input directory while naming
+      > the output directory, doing so will produce a directory name in
+      > the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is
+      > the name of the input directory. Since `<hash>` depends on the
+      > unfiltered directory, the name of the output directory will
+      > indirectly depend on files that are filtered out by the
+      > function. This will trigger a rebuild even when a filtered out
+      > file is changed. Use `builtins.path` instead, which allows
+      > specifying the name of the output directory.
+
       This function allows you to copy sources into the Nix store while
       filtering certain files. For instance, suppose that you want to use
       the directory `source-dir` as an input to a Nix expression, e.g.
@@ -1976,18 +2011,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Value * filterFun = nullptr;
     auto method = FileIngestionMethod::Recursive;
     std::optional<Hash> expectedHash;
+    PathSet context;
 
     for (auto & attr : *args[0]->attrs) {
         const string & n(attr.name);
-        if (n == "path") {
+        if (n == "path")
-            PathSet context;
             path = state.coerceToPath(*attr.pos, *attr.value, context);
-            if (!context.empty())
+        else if (attr.name == state.sName)
-                throw EvalError({
-                    .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-                    .errPos = *attr.pos
-                });
-        } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
             state.forceValue(*attr.value, pos);
@@ -2010,7 +2040,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     if (name.empty())
         name = baseNameOf(path);
 
-    addPath(state, pos, name, path, filterFun, method, expectedHash, v);
+    addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
 }
 
 static RegisterPrimOp primop_path({
@@ -2375,7 +2405,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
             .errPos = pos
         });
 
-    if (!args[0]->lambda.fun->matchAttrs) {
+    if (!args[0]->lambda.fun->hasFormals()) {
         state.mkAttrs(v, 0);
         return;
     }

src/libexpr/primops/fetchMercurial.cc

@@ -84,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
         mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
     v.attrs->sort();
 
-    if (state.allowedPaths)
+    state.allowPath(tree.storePath);
-        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -66,7 +66,7 @@ void emitTreeAttrs(
     v.attrs->sort();
 }
 
-std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
+std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
 {
     state.checkURI(uri);
     return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
@@ -81,23 +81,17 @@ std::string fixURIForGit(std::string uri, EvalState & state)
         return fixURI(uri, state);
 }
 
-void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
-{
-    string n(name);
-    attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
-}
-
 struct FetchTreeParams {
     bool emptyRevFallback = false;
     bool allowNameArgument = false;
 };
 
 static void fetchTree(
-    EvalState &state,
+    EvalState & state,
-    const Pos &pos,
+    const Pos & pos,
-    Value **args,
+    Value * * args,
-    Value &v,
+    Value & v,
-    const std::optional<std::string> type,
+    std::optional<std::string> type,
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
@@ -110,17 +104,33 @@ static void fetchTree(
 
         fetchers::Attrs attrs;
 
+        if (auto aType = args[0]->attrs->get(state.sType)) {
+            if (type)
+                throw Error({
+                    .msg = hintfmt("unexpected attribute 'type'"),
+                    .errPos = pos
+                });
+            type = state.forceStringNoCtx(*aType->value, *aType->pos);
+        } else if (!type)
+            throw Error({
+                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .errPos = pos
+            });
+
+        attrs.emplace("type", type.value());
+
         for (auto & attr : *args[0]->attrs) {
+            if (attr.name == state.sType) continue;
             state.forceValue(*attr.value);
-            if (attr.value->type() == nPath || attr.value->type() == nString)
+            if (attr.value->type() == nPath || attr.value->type() == nString) {
-                addURI(
+                auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-                    state,
+                attrs.emplace(attr.name,
-                    attrs,
+                    attr.name == "url"
-                    attr.name,
+                    ? type == "git"
-                    state.coerceToString(*attr.pos, *attr.value, context, false, false)
+                      ? fixURIForGit(s, state)
-                );
+                      : fixURI(s, state)
-            else if (attr.value->type() == nString)
+                    : s);
-                addURI(state, attrs, attr.name, attr.value->string.s);
+            }
             else if (attr.value->type() == nBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
             else if (attr.value->type() == nInt)
@@ -130,15 +140,6 @@ static void fetchTree(
                     attr.name, showType(*attr.value));
         }
 
-        if (type)
-            attrs.emplace("type", type.value());
-
-        if (!attrs.count("type"))
-            throw Error({
-                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                .errPos = pos
-            });
-
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 throw Error({
@@ -146,7 +147,6 @@ static void fetchTree(
                     .errPos = pos
                 });
 
-
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context, false, false);
@@ -169,8 +169,7 @@ static void fetchTree(
 
     auto [tree, input2] = input.fetch(state.store);
 
-    if (state.allowedPaths)
+    state.allowPath(tree.storePath);
-        state.allowedPaths->insert(tree.actualPath);
 
     emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
 }
@@ -234,19 +233,16 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
         ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
-    auto realPath = state.store->toRealPath(storePath);
-
     if (expectedHash) {
         auto hash = unpack
             ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, realPath);
+            : hashFile(htSHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash)
             throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
-    if (state.allowedPaths)
+    state.allowPath(storePath);
-        state.allowedPaths->insert(realPath);
 
     auto path = state.store->printStorePath(storePath);
     mkString(v, path, PathSet({path}));

src/libexpr/value-to-xml.cc

@@ -135,7 +135,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
             XMLOpenElement _(doc, "function", xmlAttrs);
 
-            if (v.lambda.fun->matchAttrs) {
+            if (v.lambda.fun->hasFormals()) {
                 XMLAttrs attrs;
                 if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
                 if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";

src/libstore/build/local-derivation-goal.cc

@@ -711,6 +711,7 @@ void LocalDerivationGoal::startBuilder()
     if (!builderOut.readSide)
         throw SysError("opening pseudoterminal master");
 
+    // FIXME: not thread-safe, use ptsname_r
     std::string slaveName(ptsname(builderOut.readSide.get()));
 
     if (buildUser) {
@@ -754,7 +755,6 @@ void LocalDerivationGoal::startBuilder()
     result.startTime = time(0);
 
     /* Fork a child to build the package. */
-    ProcessOptions options;
 
 #if __linux__
     if (useChroot) {
@@ -797,8 +797,6 @@ void LocalDerivationGoal::startBuilder()
 
         userNamespaceSync.create();
 
-        options.allowVfork = false;
-
         Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
         static bool userNamespacesEnabled =
             pathExists(maxUserNamespaces)
@@ -856,7 +854,7 @@ void LocalDerivationGoal::startBuilder()
             writeFull(builderOut.writeSide.get(),
                 fmt("%d %d\n", usingUserNamespace, child));
             _exit(0);
-        }, options);
+        });
 
         int res = helper.wait();
         if (res != 0 && settings.sandboxFallback) {
@@ -920,10 +918,9 @@ void LocalDerivationGoal::startBuilder()
 #endif
     {
     fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
         pid = startProcess([&]() {
             runChild();
-        }, options);
+        });
     }
 
     /* parent */
@@ -2140,8 +2137,7 @@ void LocalDerivationGoal::registerOutputs()
 
         /* Pass blank Sink as we are not ready to hash data at this stage. */
         NullSink blank;
-        auto references = worker.store.parseStorePathSet(
+        auto references = scanForReferences(blank, actualPath, referenceablePaths);
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
 
         outputReferencesIfUnregistered.insert_or_assign(
             outputName,
@@ -2208,7 +2204,7 @@ void LocalDerivationGoal::registerOutputs()
         auto rewriteOutput = [&]() {
             /* Apply hash rewriting if necessary. */
             if (!outputRewrites.empty()) {
-                warn("rewriting hashes in '%1%'; cross fingers", actualPath);
+                debug("rewriting hashes in '%1%'; cross fingers", actualPath);
 
                 /* FIXME: this is in-memory. */
                 StringSink sink;

src/libstore/build/worker.cc

@@ -239,7 +239,7 @@ void Worker::run(const Goals & _topGoals)
         }
     }
 
-    /* Call queryMissing() efficiently query substitutes. */
+    /* Call queryMissing() to efficiently query substitutes. */
     StorePathSet willBuild, willSubstitute, unknown;
     uint64_t downloadSize, narSize;
     store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);

src/libstore/references.cc

@@ -11,11 +11,13 @@
 namespace nix {
 
 
-static unsigned int refLength = 32; /* characters */
+static size_t refLength = 32; /* characters */
 
 
-static void search(const unsigned char * s, size_t len,
+static void search(
-    StringSet & hashes, StringSet & seen)
+    std::string_view s,
+    StringSet & hashes,
+    StringSet & seen)
 {
     static std::once_flag initialised;
     static bool isBase32[256];
@@ -25,7 +27,7 @@ static void search(const unsigned char * s, size_t len,
             isBase32[(unsigned char) base32Chars[i]] = true;
     });
 
-    for (size_t i = 0; i + refLength <= len; ) {
+    for (size_t i = 0; i + refLength <= s.size(); ) {
         int j;
         bool match = true;
         for (j = refLength - 1; j >= 0; --j)
@@ -35,7 +37,7 @@ static void search(const unsigned char * s, size_t len,
                 break;
             }
         if (!match) continue;
-        string ref((const char *) s + i, refLength);
+        std::string ref(s.substr(i, refLength));
         if (hashes.erase(ref)) {
             debug(format("found reference to '%1%' at offset '%2%'")
                   % ref % i);
@@ -46,69 +48,60 @@ static void search(const unsigned char * s, size_t len,
 }
 
 
-struct RefScanSink : Sink
+void RefScanSink::operator () (std::string_view data)
 {
-    StringSet hashes;
+    /* It's possible that a reference spans the previous and current
-    StringSet seen;
+       fragment, so search in the concatenation of the tail of the
+       previous fragment and the start of the current fragment. */
+    auto s = tail;
+    s.append(data.data(), refLength);
+    search(s, hashes, seen);
 
-    string tail;
+    search(data, hashes, seen);
 
-    RefScanSink() { }
+    auto tailLen = std::min(data.size(), refLength);
-
+    auto rest = refLength - tailLen;
-    void operator () (std::string_view data) override
+    if (rest < tail.size())
-    {
+        tail = tail.substr(tail.size() - rest);
-        /* It's possible that a reference spans the previous and current
+    tail.append(data.data() + data.size() - tailLen, tailLen);
-           fragment, so search in the concatenation of the tail of the
+}
-           previous fragment and the start of the current fragment. */
-        string s = tail + std::string(data, 0, refLength);
-        search((const unsigned char *) s.data(), s.size(), hashes, seen);
-
-        search((const unsigned char *) data.data(), data.size(), hashes, seen);
-
-        size_t tailLen = data.size() <= refLength ? data.size() : refLength;
-        tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
-        tail.append({data.data() + data.size() - tailLen, tailLen});
-    }
-};
 
 
-std::pair<PathSet, HashResult> scanForReferences(const string & path,
+std::pair<StorePathSet, HashResult> scanForReferences(
-    const PathSet & refs)
+    const string & path,
+    const StorePathSet & refs)
 {
     HashSink hashSink { htSHA256 };
     auto found = scanForReferences(hashSink, path, refs);
     auto hash = hashSink.finish();
-    return std::pair<PathSet, HashResult>(found, hash);
+    return std::pair<StorePathSet, HashResult>(found, hash);
 }
 
-PathSet scanForReferences(Sink & toTee,
+StorePathSet scanForReferences(
-    const string & path, const PathSet & refs)
+    Sink & toTee,
+    const Path & path,
+    const StorePathSet & refs)
 {
-    RefScanSink refsSink;
+    StringSet hashes;
-    TeeSink sink { refsSink, toTee };
+    std::map<std::string, StorePath> backMap;
-    std::map<string, Path> backMap;
 
     for (auto & i : refs) {
-        auto baseName = std::string(baseNameOf(i));
+        std::string hashPart(i.hashPart());
-        string::size_type pos = baseName.find('-');
+        auto inserted = backMap.emplace(hashPart, i).second;
-        if (pos == string::npos)
+        assert(inserted);
-            throw Error("bad reference '%1%'", i);
+        hashes.insert(hashPart);
-        string s = string(baseName, 0, pos);
-        assert(s.size() == refLength);
-        assert(backMap.find(s) == backMap.end());
-        // parseHash(htSHA256, s);
-        refsSink.hashes.insert(s);
-        backMap[s] = i;
     }
 
     /* Look for the hashes in the NAR dump of the path. */
+    RefScanSink refsSink(std::move(hashes));
+    TeeSink sink { refsSink, toTee };
     dumpPath(path, sink);
 
     /* Map the hashes found back to their store paths. */
-    PathSet found;
+    StorePathSet found;
-    for (auto & i : refsSink.seen) {
+    for (auto & i : refsSink.getResult()) {
-        std::map<string, Path>::iterator j;
+        auto j = backMap.find(i);
-        if ((j = backMap.find(i)) == backMap.end()) abort();
+        assert(j != backMap.end());
         found.insert(j->second);
     }
 

src/libstore/references.hh

@@ -1,13 +1,31 @@
 #pragma once
 
-#include "types.hh"
 #include "hash.hh"
+#include "path.hh"
 
 namespace nix {
 
-std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
+std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
 
-PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
+StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
+
+class RefScanSink : public Sink
+{
+    StringSet hashes;
+    StringSet seen;
+
+    std::string tail;
+
+public:
+
+    RefScanSink(StringSet && hashes) : hashes(hashes)
+    { }
+
+    StringSet & getResult()
+    { return seen; }
+
+    void operator () (std::string_view data) override;
+};
 
 struct RewritingSink : Sink
 {

src/libstore/tests/local.mk

@@ -0,0 +1,15 @@
+check: libstore-tests_RUN
+
+programs += libstore-tests
+
+libstore-tests_DIR := $(d)
+
+libstore-tests_INSTALL_DIR :=
+
+libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
+
+libstore-tests_LIBS = libstore libutil
+
+libstore-tests_LDFLAGS := $(GTEST_LIBS)

src/libstore/tests/references.cc

@@ -0,0 +1,45 @@
+#include "references.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+TEST(references, scan)
+{
+    std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0";
+    std::string hash2 = "zc842j0rz61mjsp3h3wp5ly71ak6qgdn";
+
+    {
+        RefScanSink scanner(StringSet{hash1});
+        auto s = "foobar";
+        scanner(s);
+        ASSERT_EQ(scanner.getResult(), StringSet{});
+    }
+
+    {
+        RefScanSink scanner(StringSet{hash1});
+        auto s = "foobar" + hash1 + "xyzzy";
+        scanner(s);
+        ASSERT_EQ(scanner.getResult(), StringSet{hash1});
+    }
+
+    {
+        RefScanSink scanner(StringSet{hash1, hash2});
+        auto s = "foobar" + hash1 + "xyzzy" + hash2;
+        scanner(((std::string_view) s).substr(0, 10));
+        scanner(((std::string_view) s).substr(10, 5));
+        scanner(((std::string_view) s).substr(15, 5));
+        scanner(((std::string_view) s).substr(20));
+        ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2}));
+    }
+
+    {
+        RefScanSink scanner(StringSet{hash1, hash2});
+        auto s = "foobar" + hash1 + "xyzzy" + hash2;
+        for (auto & i : s)
+            scanner(std::string(1, i));
+        ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2}));
+    }
+}
+
+}

src/libstore/uds-remote-store.cc

@@ -65,16 +65,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
         throw SysError("cannot create Unix domain socket");
     closeOnExec(conn->fd.get());
 
-    string socketPath = path ? *path : settings.nixDaemonSocketFile;
+    nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile);
-
-    struct sockaddr_un addr;
-    addr.sun_family = AF_UNIX;
-    if (socketPath.size() + 1 >= sizeof(addr.sun_path))
-        throw Error("socket path '%1%' is too long", socketPath);
-    strcpy(addr.sun_path, socketPath.c_str());
-
-    if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
-        throw SysError("cannot connect to daemon at '%1%'", socketPath);
 
     conn->from.fd = conn->fd.get();
     conn->to.fd = conn->fd.get();

src/libutil/archive.cc

@@ -42,7 +42,7 @@ static string caseHackSuffix = "~nix~case~hack~";
 PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
-static void dumpContents(const Path & path, size_t size,
+static void dumpContents(const Path & path, off_t size,
     Sink & sink)
 {
     sink << "contents" << size;
@@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
         sink << "type" << "regular";
         if (st.st_mode & S_IXUSR)
             sink << "executable" << "";
-        dumpContents(path, (size_t) st.st_size, sink);
+        dumpContents(path, st.st_size, sink);
     }
 
     else if (S_ISDIR(st.st_mode)) {

src/libutil/util.cc

@@ -903,7 +903,7 @@ int Pid::wait()
             return status;
         }
         if (errno != EINTR)
-            throw SysError("cannot get child exit status");
+            throw SysError("cannot get exit status of PID %d", pid);
         checkInterrupt();
     }
 }
@@ -939,9 +939,6 @@ void killUser(uid_t uid)
        users to which the current process can send signals.  So we
        fork a process, switch to uid, and send a mass kill. */
 
-    ProcessOptions options;
-    options.allowVfork = false;
-
     Pid pid = startProcess([&]() {
 
         if (setuid(uid) == -1)
@@ -964,7 +961,7 @@ void killUser(uid_t uid)
         }
 
         _exit(0);
-    }, options);
+    });
 
     int status = pid.wait();
     if (status != 0)
@@ -1085,8 +1082,7 @@ void runProgram2(const RunOptions & options)
     // vfork implies that the environment of the main process and the fork will
     // be shared (technically this is undefined, but in practice that's the
     // case), so we can't use it if we alter the environment
-    if (options.environment)
+    processOptions.allowVfork = !options.environment;
-        processOptions.allowVfork = false;
 
     /* Fork. */
     Pid pid = startProcess([&]() {
@@ -1686,16 +1682,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
 
     closeOnExec(fdSocket.get());
 
-    struct sockaddr_un addr;
+    bind(fdSocket.get(), path);
-    addr.sun_family = AF_UNIX;
-    if (path.size() + 1 >= sizeof(addr.sun_path))
-        throw Error("socket path '%1%' is too long", path);
-    strcpy(addr.sun_path, path.c_str());
-
-    unlink(path.c_str());
-
-    if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
-        throw SysError("cannot bind to socket '%1%'", path);
 
     if (chmod(path.c_str(), mode) == -1)
         throw SysError("changing permissions on '%1%'", path);
@@ -1707,6 +1694,66 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
 }
 
 
+void bind(int fd, const std::string & path)
+{
+    unlink(path.c_str());
+
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+
+    if (path.size() + 1 >= sizeof(addr.sun_path)) {
+        Pid pid = startProcess([&]() {
+            auto dir = dirOf(path);
+            if (chdir(dir.c_str()) == -1)
+                throw SysError("chdir to '%s' failed", dir);
+            std::string base(baseNameOf(path));
+            if (base.size() + 1 >= sizeof(addr.sun_path))
+                throw Error("socket path '%s' is too long", base);
+            strcpy(addr.sun_path, base.c_str());
+            if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                throw SysError("cannot bind to socket '%s'", path);
+            _exit(0);
+        });
+        int status = pid.wait();
+        if (status != 0)
+            throw Error("cannot bind to socket '%s'", path);
+    } else {
+        strcpy(addr.sun_path, path.c_str());
+        if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+            throw SysError("cannot bind to socket '%s'", path);
+    }
+}
+
+
+void connect(int fd, const std::string & path)
+{
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+
+    if (path.size() + 1 >= sizeof(addr.sun_path)) {
+        Pid pid = startProcess([&]() {
+            auto dir = dirOf(path);
+            if (chdir(dir.c_str()) == -1)
+                throw SysError("chdir to '%s' failed", dir);
+            std::string base(baseNameOf(path));
+            if (base.size() + 1 >= sizeof(addr.sun_path))
+                throw Error("socket path '%s' is too long", base);
+            strcpy(addr.sun_path, base.c_str());
+            if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                throw SysError("cannot connect to socket at '%s'", path);
+            _exit(0);
+        });
+        int status = pid.wait();
+        if (status != 0)
+            throw Error("cannot connect to socket at '%s'", path);
+    } else {
+        strcpy(addr.sun_path, path.c_str());
+        if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+            throw SysError("cannot connect to socket at '%s'", path);
+    }
+}
+
+
 string showBytes(uint64_t bytes)
 {
     return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));

src/libutil/util.hh

@@ -259,10 +259,10 @@ void killUser(uid_t uid);
    pid to the caller. */
 struct ProcessOptions
 {
-    string errorPrefix = "error: ";
+    string errorPrefix = "";
     bool dieWithParent = true;
     bool runExitHandlers = false;
-    bool allowVfork = true;
+    bool allowVfork = false;
 };
 
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
@@ -574,6 +574,12 @@ void commonChildInit(Pipe & logPipe);
 /* Create a Unix domain socket in listen mode. */
 AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
 
+/* Bind a Unix domain socket to a path. */
+void bind(int fd, const std::string & path);
+
+/* Connect to a Unix domain socket. */
+void connect(int fd, const std::string & path);
+
 
 // A Rust/Python-like enumerate() iterator adapter.
 // Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.

src/nix/daemon.cc

@@ -156,9 +156,6 @@ static void daemonLoop()
     if (chdir("/") == -1)
         throw SysError("cannot change current directory");
 
-    //  Get rid of children automatically; don't let them become zombies.
-    setSigChldAction(true);
-
     AutoCloseFD fdSocket;
 
     //  Handle socket-based activation by systemd.
@@ -176,6 +173,9 @@ static void daemonLoop()
         fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666);
     }
 
+    //  Get rid of children automatically; don't let them become zombies.
+    setSigChldAction(true);
+
     //  Loop accepting connections.
     while (1) {
 

src/nix/flake.cc

@@ -346,10 +346,10 @@ struct CmdFlakeCheck : FlakeCommand
         auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) {
             try {
                 state->forceValue(v, pos);
-                if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
+                if (!v.isLambda() || v.lambda.fun->hasFormals() || std::string(v.lambda.fun->arg) != "final")
                     throw Error("overlay does not take an argument named 'final'");
                 auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
-                if (!body || body->matchAttrs || std::string(body->arg) != "prev")
+                if (!body || body->hasFormals() || std::string(body->arg) != "prev")
                     throw Error("overlay does not take an argument named 'prev'");
                 // FIXME: if we have a 'nixpkgs' input, use it to
                 // evaluate the overlay.
@@ -363,7 +363,7 @@ struct CmdFlakeCheck : FlakeCommand
             try {
                 state->forceValue(v, pos);
                 if (v.isLambda()) {
-                    if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
+                    if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis)
                         throw Error("module must match an open attribute set ('{ config, ... }')");
                 } else if (v.type() == nAttrs) {
                     for (auto & attr : *v.attrs)

src/nix/flake.md

@@ -225,7 +225,7 @@ Currently the `type` attribute can be one of the following:
   [flake:]<flake-id>(/<rev-or-ref>(/rev)?)?
   ```
 
-  These perform a lookup of `<flake-id>` in the flake registry. or
+  These perform a lookup of `<flake-id>` in the flake registry. For
   example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake
   references. The specified `rev` and/or `ref` are merged with the
   entry in the registry; see [nix registry](./nix3-registry.md) for

src/nix/main.cc

@@ -187,11 +187,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
             , "/"),
         *vUtils);
 
-    auto vJson = state.allocValue();
+    auto vArgs = state.allocValue();
+    state.mkAttrs(*vArgs, 16);
+    auto vJson = state.allocAttr(*vArgs, state.symbols.create("command"));
     mkString(*vJson, toplevel.toJSON().dump());
+    vArgs->attrs->sort();
 
     auto vRes = state.allocValue();
-    state.callFunction(*vGenerateManpage, *vJson, *vRes, noPos);
+    state.callFunction(*vGenerateManpage, *vArgs, *vRes, noPos);
 
     auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
     if (!attr)

src/nix/profile.cc

@@ -98,10 +98,8 @@ struct ProfileManifest
 
         else if (pathExists(profile + "/manifest.nix")) {
             // FIXME: needed because of pure mode; ugly.
-            if (state.allowedPaths) {
+            state.allowPath(state.store->followLinksToStore(profile));
-                state.allowedPaths->insert(state.store->followLinksToStore(profile));
+            state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix"));
-                state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix"));
-            }
 
             auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile));
 

tests/flakes.sh

@@ -5,11 +5,6 @@ if [[ -z $(type -p git) ]]; then
     exit 99
 fi
 
-if [[ -z $(type -p hg) ]]; then
-    echo "Mercurial not installed; skipping flake tests"
-    exit 99
-fi
-
 clearStore
 rm -rf $TEST_HOME/.cache $TEST_HOME/.config
 
@@ -266,6 +261,8 @@ cat > $flake3Dir/flake.nix <<EOF
       mkDerivation {
         inherit system;
         name = "fnord";
+        dummy = builtins.readFile (builtins.path { name = "source"; path = ./.; filter = path: type: baseNameOf path == "config.nix"; } + "/config.nix");
+        dummy2 = builtins.readFile (builtins.path { name = "source"; path = inputs.flake1; filter = path: type: baseNameOf path == "simple.nix"; } + "/simple.nix");
         buildCommand = ''
           cat \${inputs.nonFlake}/README.md > \$out
         '';
@@ -579,45 +576,52 @@ nix build -o $TEST_ROOT/result git+file://$flakeGitBare
 
 # Test Mercurial flakes.
 rm -rf $flake5Dir
-hg init $flake5Dir
+mkdir $flake5Dir
 
 cat > $flake5Dir/flake.nix <<EOF
 {
   outputs = { self, flake1 }: {
     defaultPackage.$system = flake1.defaultPackage.$system;
-
     expr = assert builtins.pathExists ./flake.lock; 123;
   };
 }
 EOF
 
-hg add $flake5Dir/flake.nix
+if [[ -n $(type -p hg) ]]; then
-hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit'
+    hg init $flake5Dir
 
-nix build -o $TEST_ROOT/result hg+file://$flake5Dir
+    hg add $flake5Dir/flake.nix
-[[ -e $TEST_ROOT/result/hello ]]
+    hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit'
 
-(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
+    nix build -o $TEST_ROOT/result hg+file://$flake5Dir
+    [[ -e $TEST_ROOT/result/hello ]]
 
-nix eval hg+file://$flake5Dir#expr
+    (! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
 
-nix eval hg+file://$flake5Dir#expr
+    nix eval hg+file://$flake5Dir#expr
 
-(! nix eval hg+file://$flake5Dir#expr --no-allow-dirty)
+    nix eval hg+file://$flake5Dir#expr
 
-(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
+    (! nix eval hg+file://$flake5Dir#expr --no-allow-dirty)
 
-hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file'
+    (! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
 
-nix flake metadata --json hg+file://$flake5Dir --refresh | jq -e -r .revision
+    hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file'
-nix flake metadata --json hg+file://$flake5Dir
-[[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
 
-nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty
+    nix flake metadata --json hg+file://$flake5Dir --refresh | jq -e -r .revision
-nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-use-registries --no-allow-dirty
+    nix flake metadata --json hg+file://$flake5Dir
+    [[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
 
-# Test tarball flakes
+    nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty
-tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT --exclude .hg flake5
+    nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-use-registries --no-allow-dirty
+fi
+
+# Test path flakes.
+rm -rf $flake5Dir/.hg $flake5Dir/flake.lock
+nix flake lock path://$flake5Dir
+
+# Test tarball flakes.
+tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake5
 
 nix build -o $TEST_ROOT/result file://$TEST_ROOT/flake.tar.gz
 
@@ -632,8 +636,8 @@ nix build -o $TEST_ROOT/result "file://$TEST_ROOT/flake.tar.gz?narHash=sha256-qQ
 
 # Test --override-input.
 git -C $flake3Dir reset --hard
-nix flake lock $flake3Dir --override-input flake2/flake1 flake5 -vvvvv
+nix flake lock $flake3Dir --override-input flake2/flake1 file://$TEST_ROOT/flake.tar.gz -vvvvv
-[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake5 ]]
+[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake.tar.gz ]]
 
 nix flake lock $flake3Dir --override-input flake2/flake1 flake1
 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]

tests/nss-preload.nix

@@ -0,0 +1,123 @@
+{ nixpkgs, system, overlay }:
+
+with import (nixpkgs + "/nixos/lib/testing-python.nix") {
+  inherit system;
+  extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
+};
+
+makeTest (
+
+rec {
+  name = "nss-preload";
+
+  nodes = {
+    http_dns = { lib, pkgs, config, ... }: {
+      networking.firewall.enable = false;
+      networking.interfaces.eth1.ipv6.addresses = lib.mkForce [
+        { address = "fd21::1"; prefixLength = 64; }
+      ];
+      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
+        { address = "192.168.0.1"; prefixLength = 24; }
+      ];
+
+      services.unbound = {
+        enable = true;
+        enableRootTrustAnchor = false;
+        settings = {
+          server = {
+            interface = [ "192.168.0.1" "fd21::1" "::1" "127.0.0.1" ];
+            access-control = [ "192.168.0.0/24 allow" "fd21::/64 allow" "::1 allow" "127.0.0.0/8 allow" ];
+            local-data = [
+              ''"example.com. IN A 192.168.0.1"''
+              ''"example.com. IN AAAA fd21::1"''
+              ''"tarballs.nixos.org. IN A 192.168.0.1"''
+              ''"tarballs.nixos.org. IN AAAA fd21::1"''
+            ];
+          };
+        };
+      };
+
+      services.nginx = {
+        enable = true;
+        virtualHosts."example.com" = {
+          root = pkgs.runCommand "testdir" {} ''
+            mkdir "$out"
+            echo hello world > "$out/index.html"
+          '';
+        };
+      };
+    };
+
+    # client consumes a remote resolver
+    client = { lib, nodes, pkgs, ... }: {
+      networking.useDHCP = false;
+      networking.nameservers = [
+        (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address
+        (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address
+      ];
+      networking.interfaces.eth1.ipv6.addresses = [
+        { address = "fd21::10"; prefixLength = 64; }
+      ];
+      networking.interfaces.eth1.ipv4.addresses = [
+        { address = "192.168.0.10"; prefixLength = 24; }
+      ];
+
+      nix.sandboxPaths = lib.mkForce [];
+      nix.binaryCaches = lib.mkForce [];
+      nix.useSandbox = lib.mkForce true;
+    };
+  };
+
+  nix-fetch = pkgs.writeText "fetch.nix" ''
+    derivation {
+        # This derivation is an copy from what is available over at
+        # nix.git:corepkgs/fetchurl.nix
+        builder = "builtin:fetchurl";
+
+        # We're going to fetch data from the http_dns instance created before
+        # we expect the content to be the same as the content available there.
+        # ```
+        # $ nix-hash --type sha256 --to-base32 $(echo "hello world" | sha256sum | cut -d " " -f 1)
+        # 0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59
+        # ```
+        outputHash = "0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59";
+        outputHashAlgo = "sha256";
+        outputHashMode = "flat";
+
+        name = "example.com";
+        url = "http://example.com";
+
+        unpack = false;
+        executable = false;
+
+        system = "builtin";
+
+        preferLocalBuild = true;
+
+        impureEnvVars = [
+            "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
+        ];
+
+        urls = [ "http://example.com" ];
+      }
+  '';
+
+  testScript = { nodes, ... }: ''
+    http_dns.wait_for_unit("nginx")
+    http_dns.wait_for_open_port(80)
+    http_dns.wait_for_unit("unbound")
+    http_dns.wait_for_open_port(53)
+
+    client.start()
+    client.wait_for_unit('multi-user.target')
+
+    with subtest("can fetch data from a remote server outside sandbox"):
+        client.succeed("nix --version >&2")
+        client.succeed("curl -vvv http://example.com/index.html >&2")
+
+    with subtest("nix-build can lookup dns and fetch data"):
+        client.succeed("""
+          nix-build ${nix-fetch} >&2
+          """)
+  '';
+})