Merge branch 'no-manifests'

.gitignore

@@ -35,8 +35,11 @@ Makefile.in
 
 # /doc/manual/
 /doc/manual/manual.html
+/doc/manual/manual.xmli
+/doc/manual/manual.pdf
 /doc/manual/manual.is-valid
 /doc/manual/*.1
+/doc/manual/*.5
 /doc/manual/*.8
 /doc/manual/images
 /doc/manual/version.txt
@@ -60,6 +63,7 @@ Makefile.in
 /scripts/GeneratePatches.pm
 /scripts/download-using-manifests.pl
 /scripts/copy-from-other-stores.pl
+/scripts/download-from-binary-cache.pl
 /scripts/find-runtime-roots.pl
 /scripts/build-remote.pl
 /scripts/nix-reduce-build

configure.ac

@@ -25,12 +25,12 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM],
 
    case "$host_os" in
      linux-gnu*)
-	# For backward compatibility, strip the `-gnu' part.
-	system="$machine_name-linux";;
+        # For backward compatibility, strip the `-gnu' part.
+        system="$machine_name-linux";;
      *)
         # Strip the version number from names such as `gnu0.3',
         # `darwin10.2.0', etc.
-	system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";;
+        system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";;
    esac])
 
 sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
@@ -178,6 +178,7 @@ NEED_PROG(perl, perl)
 NEED_PROG(sed, sed)
 NEED_PROG(tar, tar)
 NEED_PROG(bzip2, bzip2)
+NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(dblatex, dblatex)
 AC_PATH_PROG(gzip, gzip)
@@ -266,7 +267,7 @@ if test "$gc" = yes; then
 fi
 
 
-# Check for the required Perl dependencies (DBI and DBD::SQLite).
+# Check for the required Perl dependencies (DBI, DBD::SQLite and WWW::Curl).
 perlFlags="-I$perllibdir"
 
 AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
@@ -277,6 +278,10 @@ AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH],
   [prefix of the Perl DBD::SQLite library]),
   perlFlags="$perlFlags -I$withval")
 
+AC_ARG_WITH(www-curl, AC_HELP_STRING([--with-www-curl=PATH],
+  [prefix of the Perl WWW::Curl library]),
+  perlFlags="$perlFlags -I$withval")
+
 AC_MSG_CHECKING([whether DBD::SQLite works])
 if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
     AC_MSG_RESULT(no)
@@ -284,6 +289,13 @@ if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
 fi
 AC_MSG_RESULT(yes)
 
+AC_MSG_CHECKING([whether WWW::Curl works])
+if ! $perl $perlFlags -e 'use WWW::Curl;' 2>&5; then
+    AC_MSG_RESULT(no)
+    AC_MSG_FAILURE([The Perl module WWW::Curl is missing.])
+fi
+AC_MSG_RESULT(yes)
+
 AC_SUBST(perlFlags)
 
 
@@ -327,6 +339,18 @@ eval dynlib_suffix=$shrext_cmds
 AC_SUBST(dynlib_suffix)
 
 
+# Do we have GNU tar?
+AC_MSG_CHECKING([if you have GNU tar])
+if $tar --version 2> /dev/null | grep -q GNU; then
+    AC_MSG_RESULT(yes)
+    tarFlags="--warning=no-timestamp"
+else
+    AC_MSG_RESULT(no)
+fi
+AC_SUBST(tarFlags)
+
+
+
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    src/Makefile

corepkgs/Makefile.am

@@ -1,6 +1,6 @@
 all-local: config.nix
 
-files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix \
+files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix derivation.nix fetchurl.nix \
 	imported-drv-to-derivation.nix
 
 install-exec-local:

corepkgs/config.nix.in

@@ -6,8 +6,10 @@ in {
   perl = "@perl@";
   shell = "@shell@";
   coreutils = "@coreutils@";
-  bzip2 = fromEnv "NIX_BZIP2" "@bzip2@";
+  bzip2 = "@bzip2@";
+  xz = "@xz@";
   tar = "@tar@";
+  tarFlags = "@tarFlags@";
   tr = "@tr@";
   curl = "@curl@";
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";

corepkgs/nar.nix

@@ -6,28 +6,37 @@ let
     ''
       export PATH=${nixBinDir}:${coreutils}
 
+      if [ $compressionType = "xz" ]; then
+        ext=xz
+        compressor="${xz} -9"
+      else
+        ext=bz2
+        compressor="${bzip2}"
+      fi
+
       echo "packing ‘$storePath’..."
       mkdir $out
-      dst=$out/tmp.nar.bz2
+      dst=$out/tmp.nar.$ext
 
       set -o pipefail
-      nix-store --dump "$storePath" | ${bzip2} > $dst
+      nix-store --dump "$storePath" | $compressor > $dst
 
-      nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
+      hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
+      echo -n $hash > $out/nar-compressed-hash
 
-      mv $out/tmp.nar.bz2 $out/$(cat $out/narbz2-hash).nar.bz2
+      mv $dst $out/$hash.nar.$ext
     '';
 
 in
 
-{ storePath, hashAlgo }:
+{ storePath, hashAlgo, compressionType }:
 
 derivation {
   name = "nar";
   system = builtins.currentSystem;
   builder = shell;
   args = [ "-e" builder ];
-  inherit storePath hashAlgo;
+  inherit storePath hashAlgo compressionType;
 
   # Don't build in a chroot because Nix's dependencies may not be there.
   __noChroot = true;

corepkgs/unpack-channel.nix

@@ -1,12 +1,29 @@
 with import <nix/config.nix>;
 
-{ name, channelName, src }:
+let
+
+  builder = builtins.toFile "unpack-channel.sh"
+    ''
+      mkdir $out
+      cd $out
+      ${bzip2} -d < $src | ${tar} xf - --warning=no-timestamp
+      mv * $out/$channelName
+      if [ -n "$binaryCacheURL" ]; then
+        mkdir $out/binary-caches
+        echo -n "$binaryCacheURL" > $out/binary-caches/$channelName
+      fi
+    '';
+
+in
+
+{ name, channelName, src, binaryCacheURL ? "" }:
 
 derivation {
   system = builtins.currentSystem;
   builder = shell;
-  args = [ "-e" ./unpack-channel.sh ];
-  inherit name channelName src bzip2 tar tr;
+  args = [ "-e" builder ];
+  inherit name channelName src binaryCacheURL;
+
   PATH = "${nixBinDir}:${coreutils}";
 
   # No point in doing this remotely.

corepkgs/unpack-channel.sh

@@ -1,4 +0,0 @@
-mkdir $out
-cd $out
-$bzip2 -d < $src | $tar xf -
-mv * $out/$channelName

doc/manual/conf-file.xml

@@ -30,6 +30,9 @@ gc-keep-derivations = true   # Idem
 env-keep-derivations = false
 </programlisting>
 
+<para>You can override settings using the <option>--option</option>
+flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
+
 <para>The following settings are currently available: 
 
 <variablelist>
@@ -243,6 +246,16 @@ env-keep-derivations = false
   </varlistentry>
 
     
+  <varlistentry><term><literal>build-fallback</literal></term>
+
+    <listitem><para>If set to <literal>true</literal>, Nix will fall
+    back to building from source if a binary substitute fails.  This
+    is equivalent to the <option>--fallback</option> flag.  The
+    default is <literal>false</literal>.</para></listitem>
+
+  </varlistentry>
+
+    
   <varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
 
     <listitem><para>When builds are performed in a chroot environment,
@@ -307,6 +320,50 @@ build-use-chroot = /dev /proc /bin</programlisting>
   </varlistentry>
 
 
+  <varlistentry><term><literal>binary-caches</literal></term>
+
+    <listitem><para>A list of URLs of binary caches, separated by
+    whitespace.  The default is empty.<!-- The default is
+    <literal>http://nixos.org/binary-cache</literal>. --></para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><literal>binary-caches-files</literal></term>
+
+    <listitem><para>A list of names of files that will be read to
+    obtain additional binary cache URLs.  The default is
+    <literal>/nix/var/nix/profiles/per-user/root/channels/binary-caches/*</literal>,
+    which ensures that Nix will use the binary caches corresponding to
+    the channels installed by root.  Do not set this option to read
+    files created by untrusted users!</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><literal>trusted-binary-caches</literal></term>
+
+    <listitem><para>A list of URLs of binary caches, separated by
+    whitespace.  These are not used by default, but can be enabled by
+    users of the Nix daemon by specifying <literal>--option
+    binary-caches <replaceable>urls</replaceable></literal> on the
+    command line.  Daemon users are only allowed to pass a subset of
+    the URLs listed in <literal>binary-caches</literal> and
+    <literal>trusted-binary-caches</literal>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
+
+    <listitem><para>The maximum number of parallel HTTP connections
+    used by the binary cache substituter to get NAR info files.  This
+    number should be high to minimise latency.  It defaults to
+    150.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>system</literal></term>
 
     <listitem><para>This option specifies the canonical Nix system

doc/manual/release-notes.xml

@@ -94,6 +94,11 @@
     <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
   </listitem>
 
+  <listitem>
+    <para>When using the Nix daemon, the <option>-s</option> flag in
+    <command>nix-env -qa</command> is now much faster.</para>
+  </listitem>
+    
 </itemizedlist>
 
 </section>

nix.spec.in

@@ -16,6 +16,7 @@ BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 %endif
 BuildRequires: perl(DBD::SQLite)
 BuildRequires: perl(DBI)
+BuildRequires: perl(WWW::Curl)
 BuildRequires: perl(ExtUtils::ParseXS)
 Requires: /usr/bin/perl
 Requires: curl

perl/Makefile.am

@@ -1,4 +1,4 @@
-PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in
+PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in lib/Nix/Utils.pm
 
 all: $(PERL_MODULES:.in=)
 

perl/lib/Nix/Config.pm.in

@@ -1,27 +1,39 @@
 package Nix::Config;
 
+$version = "@version@";
+
 $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 $libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@";
 $stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix";
 $manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests";
 $logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix";
 $confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix";
+$storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@";
 
-$bzip2 = $ENV{"NIX_BZIP2"} || "@bzip2@";
+$bzip2 = "@bzip2@";
+$xz = "@xz@";
 $curl = "@curl@";
 
 $useBindings = "@perlbindings@" eq "yes";
 
+%config = ();
+
 sub readConfig {
-    my %config;
-    my $config = "@sysconfdir@/nix/nix.conf";
+    if (defined $ENV{'_NIX_OPTIONS'}) {
+        foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) {
+            my ($n, $v) = split '=', $s, 2;
+            $config{$n} = $v;
+        }
+        return;
+    }
+
+    my $config = "$confDir/nix.conf";
     return unless -f $config;
 
     open CONFIG, "<$config" or die "cannot open `$config'";
     while (<CONFIG>) {
         /^\s*([\w|-]+)\s*=\s*(.*)$/ or next;
         $config{$1} = $2;
-        print "|$1| -> |$2|\n";
     }
     close CONFIG;
 }

perl/lib/Nix/Store.xs

@@ -19,7 +19,7 @@ void doInit()
 {
     if (!store) {
         try {
-            setDefaultsFromEnvironment();
+            settings.processEnvironment();
             store = openStore();
         } catch (Error & e) {
             croak(e.what());

perl/lib/Nix/Utils.pm

@@ -0,0 +1,19 @@
+package Nix::Utils;
+
+$urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )";
+
+sub checkURL {
+    my ($url) = @_;
+    die "invalid URL ‘$url’\n" unless $url =~ /^ $urlRE $ /x;
+}
+
+sub uniq {
+    my %seen;
+    my @res;
+    foreach my $name (@_) {
+        next if $seen{$name};
+        $seen{$name} = 1;
+        push @res, $name;
+    }
+    return @res;
+}

release.nix

@@ -29,6 +29,7 @@ let
           --with-xml-flags=--nonet
           --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
           --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
+          --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
         '';
 
         postUnpack = ''
@@ -77,6 +78,7 @@ let
           --disable-init-state
           --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
           --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
+          --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
           --enable-gc
         '';
 
@@ -134,6 +136,7 @@ let
           --disable-init-state
           --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
           --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
+          --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
         '';
 
         dontInstall = false;
@@ -192,7 +195,7 @@ let
       name = "nix-rpm-${diskImage.name}";
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" ]; };
+        { extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" ]; };
       memSize = 1024;
       meta.schedulingPriority = prio;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
@@ -211,7 +214,7 @@ let
       name = "nix-deb";
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" ]; };
+        { extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" ]; };
       memSize = 1024;
       meta.schedulingPriority = prio;
       configureFlags = "--sysconfdir=/etc";

scripts/Makefile.am

@@ -7,17 +7,14 @@ noinst_SCRIPTS = nix-profile.sh \
   find-runtime-roots.pl build-remote.pl nix-reduce-build \
   copy-from-other-stores.pl nix-http-export.cgi
 
-nix-pull nix-push: download-using-manifests.pl
-
-install-exec-local: download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
+install-exec-local: download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl find-runtime-roots.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
-	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
-	$(INSTALL_PROGRAM) copy-from-other-stores.pl $(DESTDIR)$(libexecdir)/nix/substituters
+	$(INSTALL_PROGRAM) download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 
 include ../substitute.mk
@@ -29,6 +26,7 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-build.in \
   download-using-manifests.pl.in \
   copy-from-other-stores.pl.in \
+  download-from-binary-cache.pl.in \
   nix-copy-closure.in \
   find-runtime-roots.pl.in \
   build-remote.pl.in \

scripts/copy-from-other-stores.pl.in

@@ -36,42 +36,45 @@ sub findStorePath {
 if ($ARGV[0] eq "--query") {
 
     while (<STDIN>) {
-        my $cmd = $_; chomp $cmd;
+        chomp;
+        my ($cmd, @args) = split " ", $_;
 
         if ($cmd eq "have") {
-            my $storePath = <STDIN>; chomp $storePath;
-            print STDOUT (defined findStorePath($storePath) ? "1\n" : "0\n");
+            foreach my $storePath (@args) {
+                print "$storePath\n" if defined findStorePath($storePath);
+            }
+            print "\n";
         }
 
         elsif ($cmd eq "info") {
-            my $storePath = <STDIN>; chomp $storePath;
-            my ($store, $sourcePath) = findStorePath($storePath);
-            if (!defined $store) {
-                print "0\n";
-                next; # not an error
+            foreach my $storePath (@args) {
+                my ($store, $sourcePath) = findStorePath($storePath);
+                next unless defined $store;
+
+                $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
+
+                my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
+                die "cannot query deriver of `$storePath'" if $? != 0;
+                chomp $deriver;
+                $deriver = "" if $deriver eq "unknown-deriver";
+
+                my @references = split "\n",
+                    `@bindir@/nix-store --query --references $storePath`;
+                die "cannot query references of `$storePath'" if $? != 0;
+
+                my $narSize = `@bindir@/nix-store --query --size $storePath`;
+                die "cannot query size of `$storePath'" if $? != 0;
+                chomp $narSize;
+
+                print "$storePath\n";
+                print "$deriver\n";
+                print scalar @references, "\n";
+                print "$_\n" foreach @references;
+                print "$narSize\n";
+                print "$narSize\n";
             }
-            print "1\n";
 
-            $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
-            
-            my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
-            die "cannot query deriver of `$storePath'" if $? != 0;
-            chomp $deriver;
-            $deriver = "" if $deriver eq "unknown-deriver";
-
-            my @references = split "\n",
-                `@bindir@/nix-store --query --references $storePath`;
-            die "cannot query references of `$storePath'" if $? != 0;
-
-            my $narSize = `@bindir@/nix-store --query --size $storePath`;
-            die "cannot query size of `$storePath'" if $? != 0;
-            chomp $narSize;
-
-            print "$deriver\n";
-            print scalar @references, "\n";
-            print "$_\n" foreach @references;
-            print "$narSize\n";
-            print "$narSize\n";
+            print "\n";
         }
 
         else { die "unknown command `$cmd'"; }
@@ -84,9 +87,10 @@ elsif ($ARGV[0] eq "--substitute") {
     my $storePath = $ARGV[1];
     my ($store, $sourcePath) = findStorePath $storePath;
     die unless $store;
-    print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
+    print STDERR "\n*** Copying `$storePath' from `$sourcePath'\n\n";
     system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
         or die "cannot copy `$sourcePath' to `$storePath'";
+    print "\n"; # no hash to verify
 }
 
 

scripts/download-from-binary-cache.pl.in

@@ -0,0 +1,537 @@
+#! @perl@ -w @perlFlags@
+
+use DBI;
+use File::Basename;
+use IO::Select;
+use Nix::Config;
+use Nix::Store;
+use Nix::Utils;
+use WWW::Curl::Easy;
+use WWW::Curl::Multi;
+use strict;
+
+
+Nix::Config::readConfig;
+
+my @caches;
+my $gotCaches = 0;
+
+my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150);
+$maxParallelRequests = 1 if $maxParallelRequests < 1;
+
+my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
+
+my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence);
+
+my $curlm = WWW::Curl::Multi->new;
+my $activeRequests = 0;
+my $curlIdCount = 1;
+my %requests;
+my %scheduled;
+my $caBundle = $ENV{"CURL_CA_BUNDLE"} // $ENV{"OPENSSL_X509_CERT_FILE"};
+
+
+sub addRequest {
+    my ($storePath, $url, $head) = @_;
+
+    my $curl = WWW::Curl::Easy->new;
+    my $curlId = $curlIdCount++;
+    $requests{$curlId} = { storePath => $storePath, url => $url, handle => $curl, content => "", type => $head ? "HEAD" : "GET" };
+
+    $curl->setopt(CURLOPT_PRIVATE, $curlId);
+    $curl->setopt(CURLOPT_URL, $url);
+    $curl->setopt(CURLOPT_WRITEDATA, \$requests{$curlId}->{content});
+    $curl->setopt(CURLOPT_FOLLOWLOCATION, 1);
+    $curl->setopt(CURLOPT_CAINFO, $caBundle) if defined $caBundle;
+    $curl->setopt(CURLOPT_USERAGENT, "Nix/$Nix::Config::version");
+    $curl->setopt(CURLOPT_NOBODY, 1) if $head;
+    $curl->setopt(CURLOPT_FAILONERROR, 1);
+
+    if ($activeRequests >= $maxParallelRequests) {
+        $scheduled{$curlId} = 1;
+    } else {
+        $curlm->add_handle($curl);
+        $activeRequests++;
+    }
+
+    return $requests{$curlId};
+}
+
+
+sub processRequests {
+    while ($activeRequests) {
+        my ($rfds, $wfds, $efds) = $curlm->fdset();
+        #print STDERR "R = @{$rfds}, W = @{$wfds}, E = @{$efds}\n";
+
+        # Sleep until we can read or write some data.
+        if (scalar @{$rfds} + scalar @{$wfds} + scalar @{$efds} > 0) {
+            IO::Select->select(IO::Select->new(@{$rfds}), IO::Select->new(@{$wfds}), IO::Select->new(@{$efds}), 0.1);
+        }
+
+        if ($curlm->perform() != $activeRequests) {
+            while (my ($id, $result) = $curlm->info_read) {
+                if ($id) {
+                    my $request = $requests{$id} or die;
+                    my $handle = $request->{handle};
+                    $request->{result} = $result;
+                    $request->{httpStatus} = $handle->getinfo(CURLINFO_RESPONSE_CODE);
+
+                    print STDERR "$request->{type} on $request->{url} [$request->{result}, $request->{httpStatus}]\n" if $debug;
+
+                    $activeRequests--;
+                    delete $request->{handle};
+
+                    if (scalar(keys %scheduled) > 0) {
+                        my $id2 = (keys %scheduled)[0];
+                        $curlm->add_handle($requests{$id2}->{handle});
+                        $activeRequests++;
+                        delete $scheduled{$id2};
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+sub initCache {
+    my $dbPath = "$Nix::Config::stateDir/binary-cache-v1.sqlite";
+
+    # Open/create the database.
+    $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
+        or die "cannot open database `$dbPath'";
+    $dbh->{RaiseError} = 1;
+    $dbh->{PrintError} = 0;
+
+    $dbh->do("pragma synchronous = off"); # we can always reproduce the cache
+    $dbh->do("pragma journal_mode = truncate");
+
+    # Initialise the database schema, if necessary.
+    $dbh->do(<<EOF);
+        create table if not exists BinaryCaches (
+            id        integer primary key autoincrement not null,
+            url       text unique not null,
+            timestamp integer not null,
+            storeDir  text not null,
+            wantMassQuery integer not null
+        );
+EOF
+
+    $dbh->do(<<EOF);
+        create table if not exists NARs (
+            cache            integer not null,
+            storePath        text not null,
+            url              text not null,
+            compression      text not null,
+            fileHash         text,
+            fileSize         integer,
+            narHash          text,
+            narSize          integer,
+            refs             text,
+            deriver          text,
+            system           text,
+            timestamp        integer not null,
+            primary key (cache, storePath),
+            foreign key (cache) references BinaryCaches(id) on delete cascade
+        );
+EOF
+
+    $dbh->do(<<EOF);
+        create table if not exists NARExistence (
+            cache            integer not null,
+            storePath        text not null,
+            exist            integer not null,
+            timestamp        integer not null,
+            primary key (cache, storePath),
+            foreign key (cache) references BinaryCaches(id) on delete cascade
+        );
+EOF
+
+    $queryCache = $dbh->prepare("select id, storeDir, wantMassQuery from BinaryCaches where url = ?") or die;
+
+    $insertNAR = $dbh->prepare(
+        "insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " .
+        "narSize, refs, deriver, system, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
+
+    $queryNAR = $dbh->prepare("select * from NARs where cache = ? and storePath = ?") or die;
+
+    $insertNARExistence = $dbh->prepare(
+        "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)") or die;
+
+    $queryNARExistence = $dbh->prepare("select exist from NARExistence where cache = ? and storePath = ?") or die;
+}
+
+
+sub getAvailableCaches {
+    return if $gotCaches;
+    $gotCaches = 1;
+
+    sub strToList {
+        my ($s) = @_;
+        return map { s/\/+$//; $_ } split(/ /, $s);
+    }
+
+    my @urls = strToList ($Nix::Config::config{"binary-caches"} // "");
+    # // ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : ""));
+
+    my $urlsFiles = $Nix::Config::config{"binary-cache-files"}
+        // "/nix/var/nix/profiles/per-user/root/channels/binary-caches/*";
+    foreach my $urlFile (glob $urlsFiles) {
+        next unless -f $urlFile;
+        open FILE, "<$urlFile" or die "cannot open ‘$urlFile’\n";
+        my $url = <FILE>; chomp $url;
+        close FILE;
+        push @urls, strToList($url);
+    }
+
+    # Allow Nix daemon users to override the binary caches to a subset
+    # of those listed in the config file.  Note that ‘untrusted-*’
+    # denotes options passed by the client.
+    if (defined $Nix::Config::config{"untrusted-binary-caches"}) {
+        my @untrustedUrls = strToList $Nix::Config::config{"untrusted-binary-caches"};
+        my @trustedUrls = (@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
+        @urls = ();
+        foreach my $url (@untrustedUrls) {
+            die "binary cache ‘$url’ is not trusted (please add it to ‘trusted-binary-caches’ in $Nix::Config::confDir/nix.conf)\n"
+                unless grep { $url eq $_ } @trustedUrls > 0;
+            push @urls, $url;
+        }
+    }
+
+    foreach my $url (Nix::Utils::uniq @urls) {
+
+        # FIXME: not atomic.
+        $queryCache->execute($url);
+        my $res = $queryCache->fetchrow_hashref();
+        if (defined $res) {
+            next if $res->{storeDir} ne $Nix::Config::storeDir;
+            push @caches, { id => $res->{id}, url => $url, wantMassQuery => $res->{wantMassQuery} };
+            next;
+        }
+
+        # Get the cache info file.
+        my $request = addRequest(undef, $url . "/nix-cache-info");
+        processRequests;
+
+        if ($request->{result} != 0) {
+            print STDERR "could not download ‘$request->{url}’ (" .
+                ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
+            next;
+        }
+
+        my $storeDir = "/nix/store";
+        my $wantMassQuery = 0;
+        foreach my $line (split "\n", $request->{content}) {
+            unless ($line =~ /^(.*): (.*)$/) {
+                print STDERR "bad cache info file ‘$request->{url}’\n";
+                return undef;
+            }
+            if ($1 eq "StoreDir") { $storeDir = $2; }
+            elsif ($1 eq "WantMassQuery") { $wantMassQuery = int($2); }
+        }
+
+        $dbh->do("insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery) values (?, ?, ?, ?)",
+                 {}, $url, time(), $storeDir, $wantMassQuery);
+        my $id = $dbh->last_insert_id("", "", "", "");
+        next if $storeDir ne $Nix::Config::storeDir;
+        push @caches, { id => $id, url => $url, wantMassQuery => $wantMassQuery };
+    }
+}
+
+
+sub processNARInfo {
+    my ($storePath, $cache, $request) = @_;
+
+    if ($request->{result} != 0) {
+        if ($request->{result} != 37 && $request->{httpStatus} != 404) {
+            print STDERR "could not download ‘$request->{url}’ (" .
+                ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
+        } else {
+            $insertNARExistence->execute($cache->{id}, basename($storePath), 0, time())
+                unless $request->{url} =~ /^file:/;
+        }
+        return undef;
+    }
+
+    my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system);
+    my $compression = "bzip2";
+    my @refs;
+    foreach my $line (split "\n", $request->{content}) {
+        unless ($line =~ /^(.*): (.*)$/) {
+            print STDERR "bad NAR info file ‘$request->{url}’\n";
+            return undef;
+        }
+        if ($1 eq "StorePath") { $storePath2 = $2; }
+        elsif ($1 eq "URL") { $url = $2; }
+        elsif ($1 eq "Compression") { $compression = $2; }
+        elsif ($1 eq "FileHash") { $fileHash = $2; }
+        elsif ($1 eq "FileSize") { $fileSize = int($2); }
+        elsif ($1 eq "NarHash") { $narHash = $2; }
+        elsif ($1 eq "NarSize") { $narSize = int($2); }
+        elsif ($1 eq "References") { @refs = split / /, $2; }
+        elsif ($1 eq "Deriver") { $deriver = $2; }
+        elsif ($1 eq "System") { $system = $2; }
+    }
+    return undef if $storePath ne $storePath2;
+    if ($storePath ne $storePath2 || !defined $url || !defined $narHash) {
+        print STDERR "bad NAR info file ‘$request->{url}’\n";
+        return undef;
+    }
+
+    # Cache the result.
+    $insertNAR->execute(
+        $cache->{id}, basename($storePath), $url, $compression, $fileHash, $fileSize,
+        $narHash, $narSize, join(" ", @refs), $deriver, $system, time())
+        unless $request->{url} =~ /^file:/;
+
+    return
+        { url => $url
+        , compression => $compression
+        , fileHash => $fileHash
+        , fileSize => $fileSize
+        , narHash => $narHash
+        , narSize => $narSize
+        , refs => [ @refs ]
+        , deriver => $deriver
+        , system => $system
+        };
+}
+
+
+sub getCachedInfoFrom {
+    my ($storePath, $cache) = @_;
+
+    $queryNAR->execute($cache->{id}, basename($storePath));
+    my $res = $queryNAR->fetchrow_hashref();
+    return undef unless defined $res;
+
+    return
+        { url => $res->{url}
+        , compression => $res->{compression}
+        , fileHash => $res->{fileHash}
+        , fileSize => $res->{fileSize}
+        , narHash => $res->{narHash}
+        , narSize => $res->{narSize}
+        , refs => [ split " ", $res->{refs} ]
+        , deriver => $res->{deriver}
+        } if defined $res;
+}
+
+
+sub negativeHit {
+    my ($storePath, $cache) = @_;
+    $queryNARExistence->execute($cache->{id}, basename($storePath));
+    my $res = $queryNARExistence->fetchrow_hashref();
+    return defined $res && $res->{exist} == 0;
+}
+
+
+sub positiveHit {
+    my ($storePath, $cache) = @_;
+    return 1 if defined getCachedInfoFrom($storePath, $cache);
+    $queryNARExistence->execute($cache->{id}, basename($storePath));
+    my $res = $queryNARExistence->fetchrow_hashref();
+    return defined $res && $res->{exist} == 1;
+}
+
+
+sub printInfo {
+    my ($storePath, $info) = @_;
+    print "$storePath\n";
+    print $info->{deriver} ? "$Nix::Config::storeDir/$info->{deriver}" : "", "\n";
+    print scalar @{$info->{refs}}, "\n";
+    print "$Nix::Config::storeDir/$_\n" foreach @{$info->{refs}};
+    print $info->{fileSize} || 0, "\n";
+    print $info->{narSize} || 0, "\n";
+}
+
+
+sub infoUrl {
+    my ($binaryCacheUrl, $storePath) = @_;
+    my $pathHash = substr(basename($storePath), 0, 32);
+    my $infoUrl = "$binaryCacheUrl/$pathHash.narinfo";
+}
+
+
+sub printInfoParallel {
+    my @paths = @_;
+
+    # First print all paths for which we have cached info.
+    my @left;
+    foreach my $storePath (@paths) {
+        my $found = 0;
+        foreach my $cache (@caches) {
+            my $info = getCachedInfoFrom($storePath, $cache);
+            if (defined $info) {
+                printInfo($storePath, $info);
+                $found = 1;
+                last;
+            }
+        }
+        push @left, $storePath if !$found;
+    }
+
+    return if scalar @left == 0;
+
+    foreach my $cache (@caches) {
+
+        my @left2;
+        %requests = ();
+        foreach my $storePath (@left) {
+            if (negativeHit($storePath, $cache)) {
+                push @left2, $storePath;
+                next;
+            }
+            addRequest($storePath, infoUrl($cache->{url}, $storePath));
+        }
+
+        processRequests;
+
+        foreach my $request (values %requests) {
+            my $info = processNARInfo($request->{storePath}, $cache, $request);
+            if (defined $info) {
+                printInfo($request->{storePath}, $info);
+            } else {
+                push @left2, $request->{storePath};
+            }
+        }
+
+        @left = @left2;
+    }
+}
+
+
+sub printSubstitutablePaths {
+    my @paths = @_;
+
+    # First look for paths that have cached info.
+    my @left;
+    foreach my $storePath (@paths) {
+        my $found = 0;
+        foreach my $cache (@caches) {
+            next unless $cache->{wantMassQuery};
+            if (positiveHit($storePath, $cache)) {
+                print "$storePath\n";
+                $found = 1;
+                last;
+            }
+        }
+        push @left, $storePath if !$found;
+    }
+
+    return if scalar @left == 0;
+
+    # For remaining paths, do HEAD requests.
+    foreach my $cache (@caches) {
+        next unless $cache->{wantMassQuery};
+        my @left2;
+        %requests = ();
+        foreach my $storePath (@left) {
+            if (negativeHit($storePath, $cache)) {
+                push @left2, $storePath;
+                next;
+            }
+            addRequest($storePath, infoUrl($cache->{url}, $storePath), 1);
+        }
+
+        processRequests;
+
+        foreach my $request (values %requests) {
+            if ($request->{result} != 0) {
+                if ($request->{result} != 37 && $request->{httpStatus} != 404) {
+                    print STDERR "could not check ‘$request->{url}’ (" .
+                        ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
+                } else {
+                    $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 0, time())
+                        unless $request->{url} =~ /^file:/;
+                }
+                push @left2, $request->{storePath};
+            } else {
+                $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 1, time())
+                    unless $request->{url} =~ /^file:/;
+                print "$request->{storePath}\n";
+            }
+        }
+
+        @left = @left2;
+    }
+}
+
+
+sub downloadBinary {
+    my ($storePath) = @_;
+
+    foreach my $cache (@caches) {
+        my $info = getCachedInfoFrom($storePath, $cache);
+
+        unless (defined $info) {
+            next if negativeHit($storePath, $cache);
+            my $request = addRequest($storePath, infoUrl($cache->{url}, $storePath));
+            processRequests;
+            $info = processNARInfo($storePath, $cache, $request);
+        }
+
+        next unless defined $info;
+
+        my $decompressor;
+        if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
+        elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; }
+        else {
+            print STDERR "unknown compression method ‘$info->{compression}’\n";
+            next;
+        }
+        my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
+        print STDERR "\n*** Downloading ‘$url’ into ‘$storePath’...\n";
+        Nix::Utils::checkURL $url;
+        if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) {
+            die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
+            next;
+        }
+
+        # Tell Nix about the expected hash so it can verify it.
+        print "$info->{narHash}\n";
+
+        print STDERR "\n";
+        return;
+    }
+
+    print STDERR "could not download ‘$storePath’ from any binary cache\n";
+}
+
+
+initCache();
+
+
+if ($ARGV[0] eq "--query") {
+
+    while (<STDIN>) {
+        getAvailableCaches;
+        chomp;
+        my ($cmd, @args) = split " ", $_;
+
+        if ($cmd eq "have") {
+            printSubstitutablePaths(@args);
+            print "\n";
+        }
+
+        elsif ($cmd eq "info") {
+            printInfoParallel(@args);
+            print "\n";
+        }
+
+        else { die "unknown command `$cmd'"; }
+
+        flush STDOUT;
+    }
+
+}
+
+elsif ($ARGV[0] eq "--substitute") {
+    my $storePath = $ARGV[1] or die;
+    getAvailableCaches;
+    downloadBinary($storePath);
+}
+
+else {
+    die;
+}

scripts/download-using-manifests.pl.in

@@ -4,6 +4,7 @@ use strict;
 use Nix::Config;
 use Nix::Manifest;
 use Nix::Store;
+use Nix::Utils;
 use POSIX qw(strftime);
 use File::Temp qw(tempdir);
 
@@ -15,6 +16,9 @@ my $logFile = "$Nix::Config::logDir/downloads";
 # estimating the expected download size.
 my $fast = 1;
 
+# ‘--insecure’ is fine because Nix verifies the hash of the result.
+my $curl = "$Nix::Config::curl --fail --location --insecure";
+
 
 # Open the manifest cache and update it if necessary.
 my $dbh = updateManifestDB();
@@ -173,56 +177,54 @@ sub computeSmallestDownload {
 if ($ARGV[0] eq "--query") {
 
     while (<STDIN>) {
-        my $cmd = $_; chomp $cmd;
+        chomp;
+        my ($cmd, @args) = split " ", $_;
 
         if ($cmd eq "have") {
-            my $storePath = <STDIN>; chomp $storePath;
-            print STDOUT (
-                scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0
-                ? "1\n" : "0\n");
+            foreach my $storePath (@args) {
+                print "$storePath\n" if scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0;
+            }
+            print "\n";
         }
 
         elsif ($cmd eq "info") {
-            my $storePath = <STDIN>; chomp $storePath;
+            foreach my $storePath (@args) {
 
-            my $infos = $dbh->selectall_arrayref(
-                "select * from NARs where storePath = ?",
-                { Slice => {} }, $storePath);
+                my $infos = $dbh->selectall_arrayref(
+                    "select * from NARs where storePath = ?",
+                    { Slice => {} }, $storePath);
 
-            my $info;
-            if (scalar @{$infos} > 0) {
-                $info = @{$infos}[0];
-            }
-            else {
-                print "0\n";
-                next; # not an error
-            }
+                next unless scalar @{$infos} > 0;
+                my $info = @{$infos}[0];
 
-            print "1\n";
-            print "$info->{deriver}\n";
-            my @references = split " ", $info->{refs};
-            print scalar @references, "\n";
-            print "$_\n" foreach @references;
+                print "$storePath\n";
+                print "$info->{deriver}\n";
+                my @references = split " ", $info->{refs};
+                print scalar @references, "\n";
+                print "$_\n" foreach @references;
 
-            my @path = computeSmallestDownload $storePath;
+                my @path = computeSmallestDownload $storePath;
 
-            my $downloadSize = 0;
-            while (scalar @path > 0) {
-                my $edge = pop @path;
-                my $u = $edge->{start};
-                my $v = $edge->{end};
-                if ($edge->{type} eq "patch") {
-                    $downloadSize += $edge->{info}->{size} || 0;
-                }
-                elsif ($edge->{type} eq "narfile") {
-                    $downloadSize += $edge->{info}->{size} || 0;
+                my $downloadSize = 0;
+                while (scalar @path > 0) {
+                    my $edge = pop @path;
+                    my $u = $edge->{start};
+                    my $v = $edge->{end};
+                    if ($edge->{type} eq "patch") {
+                        $downloadSize += $edge->{info}->{size} || 0;
+                    }
+                    elsif ($edge->{type} eq "narfile") {
+                        $downloadSize += $edge->{info}->{size} || 0;
+                    }
                 }
+
+                print "$downloadSize\n";
+
+                my $narSize = $info->{narSize} || 0;
+                print "$narSize\n";
             }
 
-            print "$downloadSize\n";
-            
-            my $narSize = $info->{narSize} || 0;
-            print "$narSize\n";
+            print "\n";
         }
 
         else { die "unknown command `$cmd'"; }
@@ -273,16 +275,6 @@ $dbh->disconnect;
 my $curStep = 1;
 my $maxStep = scalar @path;
 
-sub downloadFile { 
-    my $url = shift; 
-    $ENV{"PRINT_PATH"} = 1;
-    $ENV{"QUIET"} = 1;
-    my ($hash, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`;
-    die "download of `$url' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
-    chomp $path;
-    return $path;
-}
-
 my $finalNarHash;
 
 while (scalar @path > 0) {
@@ -314,13 +306,16 @@ while (scalar @path > 0) {
 
         # Download the patch.
         print STDERR "  downloading patch...\n";
-        my $patchPath = downloadFile "$patch->{url}";
+        my $patchPath = "$tmpDir/patch";
+        Nix::Utils::checkURL $patch->{url};
+        system("$curl '$patch->{url}' -o $patchPath") == 0
+            or die "cannot download patch `$patch->{url}'\n";
 
         # Apply the patch to the NAR archive produced in step 1 (for
         # the already present path) or a later step (for patch sequences).
         print STDERR "  applying patch...\n";
         system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0
-            or die "cannot apply patch `$patchPath' to $tmpNar";
+            or die "cannot apply patch `$patchPath' to $tmpNar\n";
 
         if ($curStep < $maxStep) {
             # The archive will be used as the base of the next patch.
@@ -330,7 +325,7 @@ while (scalar @path > 0) {
             # into the target path.
             print STDERR "  unpacking patched archive...\n";
             system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0
-                or die "cannot unpack $tmpNar2 into `$v'";
+                or die "cannot unpack $tmpNar2 into `$v'\n";
         }
 
         $finalNarHash = $patch->{narHash};
@@ -343,19 +338,15 @@ while (scalar @path > 0) {
         my $size = $narFile->{size} || -1;
         print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
 
-        # Download the archive.
-        print STDERR "  downloading archive...\n";
-        my $narFilePath = downloadFile "$narFile->{url}";
-
+        Nix::Utils::checkURL $narFile->{url};
         if ($curStep < $maxStep) {
             # The archive will be used a base to a patch.
-            system("$Nix::Config::bzip2 -d < '$narFilePath' > $tmpNar") == 0
-                or die "cannot unpack `$narFilePath' into `$v'";
+            system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d > $tmpNar") == 0
+                or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
         } else {
             # Unpack the archive into the target path.
-            print STDERR "  unpacking archive...\n";
-            system("$Nix::Config::bzip2 -d < '$narFilePath' | $Nix::Config::binDir/nix-store --restore '$v'") == 0
-                or die "cannot unpack `$narFilePath' into `$v'";
+            system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d | $Nix::Config::binDir/nix-store --restore '$v'") == 0
+                or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
         }
 
         $finalNarHash = $narFile->{narHash};
@@ -365,21 +356,10 @@ while (scalar @path > 0) {
 }
 
 
-# Make sure that the hash declared in the manifest matches what we
-# downloaded and unpacked.
-
-if (defined $finalNarHash) {
-    my ($hashAlgo, $hash) = parseHash $finalNarHash;
-
-    # The hash in the manifest can be either in base-16 or base-32.
-    # Handle both.
-    my $hash2 = hashPath($hashAlgo, $hashAlgo eq "sha256" && length($hash) != 64, $targetPath);
-    
-    die "hash mismatch in downloaded path $targetPath; expected $hash, got $hash2\n"
-        if $hash ne $hash2;
-} else {
-    die "cannot check integrity of the downloaded path since its hash is not known\n";
-}
+# Tell Nix about the expected hash so it can verify it.
+die "cannot check integrity of the downloaded path since its hash is not known\n"
+    unless defined $finalNarHash;
+print "$finalNarHash\n";
 
 
 print STDERR "\n";

scripts/nix-build.in

@@ -58,6 +58,11 @@ EOF
         # '` hack
     }
 
+    elsif ($arg eq "--version") {
+        print "nix-build (Nix) $Nix::Config::version\n";
+        exit 0;
+    }
+    
     elsif ($arg eq "--add-drv-link") {
         $drvLink = "./derivation";
     }

scripts/nix-channel.in

@@ -80,12 +80,6 @@ sub update {
 
     readChannels;
 
-    # Create the manifests directory if it doesn't exist.
-    mkdir $manifestDir, 0755 unless -e $manifestDir;
-
-    # Do we have write permission to the manifests directory?
-    die "$0: you do not have write permission to `$manifestDir'!\n" unless -W $manifestDir;
-
     # Download each channel.
     my $exprs = "";
     foreach my $name (keys %channels) {
@@ -102,10 +96,19 @@ sub update {
         $headers =~ s/\r//g;
         $url = $1 if $headers =~ /^Location:\s*(.*)\s*$/m;
 
-        # Pull the channel manifest.
-        $ENV{'NIX_ORIG_URL'} = $origUrl;
-        system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
-            or die "cannot pull manifest from `$url'\n";
+        # Check if the channel advertises a binary cache.
+        my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
+        my $extraAttrs = "";
+        if ($? == 0 && $binaryCacheURL ne "") {
+            $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; ";
+        } else {
+            # No binary cache, so pull the channel manifest.
+            mkdir $manifestDir, 0755 unless -e $manifestDir;
+            die "$0: you do not have write permission to `$manifestDir'!\n" unless -W $manifestDir;
+            $ENV{'NIX_ORIG_URL'} = $origUrl;
+            system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
+                or die "cannot pull manifest from `$url'\n";
+        }
 
         # Download the channel tarball.
         my $fullURL = "$url/nixexprs.tar.bz2";
@@ -120,7 +123,7 @@ sub update {
         my $cname = $name;
         $cname .= $1 if basename($url) =~ /(-\d.*)$/;
 
-        $exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }' ";
+        $exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' ";
     }
 
     # Unpack the channel tarballs into the Nix store and install them
@@ -194,6 +197,11 @@ while (scalar @ARGV) {
         usageError;
     }
 
+    elsif ($arg eq "--version") {
+        print "nix-channel (Nix) $Nix::Config::version\n";
+        exit 0;
+    }
+
     else {
         die "unknown argument `$arg'; try `--help'";
     }

scripts/nix-install-package.in

@@ -3,6 +3,7 @@
 use strict;
 use File::Temp qw(tempdir);
 use Nix::Config;
+use Nix::Utils;
 
 
 sub usageError {
@@ -72,7 +73,7 @@ my $tmpDir = tempdir("nix-install-package.XXXXXX", CLEANUP => 1, TMPDIR => 1)
 
 sub barf {
     my $msg = shift;
-    print "$msg\n";
+    print "\nInstallation failed: $msg\n";
     <STDIN> if $interactive;
     exit 1;
 }
@@ -92,7 +93,6 @@ open PKGFILE, "<$pkgFile" or barf "cannot open `$pkgFile': $!";
 my $contents = <PKGFILE>;
 close PKGFILE;
 
-my $urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ )";
 my $nameRE = "(?: [A-Za-z0-9\+\-\.\_\?\=]+ )"; # see checkStoreName()
 my $systemRE = "(?: [A-Za-z0-9\+\-\_]+ )";
 my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
@@ -101,7 +101,7 @@ my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
 # store path.  We'll let nix-env do that.
 
 $contents =~
-    / ^ \s* (\S+) \s+ ($urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) /x
+    / ^ \s* (\S+) \s+ ($Nix::Utils::urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )?  /x
     or barf "invalid package contents";
 my $version = $1;
 my $manifestURL = $2;
@@ -109,6 +109,7 @@ my $drvName = $3;
 my $system = $4;
 my $drvPath = $5;
 my $outPath = $6;
+my $binaryCacheURL = $8;
 
 barf "invalid package version `$version'" unless $version eq "NIXPKG1";
 
@@ -122,17 +123,25 @@ if ($interactive) {
 }
 
 
-# Store the manifest in the temporary directory so that we don't
-# pollute /nix/var/nix/manifests.  This also requires that we don't
-# use the Nix daemon (because otherwise download-using-manifests won't
-# see our NIX_MANIFESTS_DIRS environment variable).
-$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
-$ENV{NIX_REMOTE} = "";
+if (defined $binaryCacheURL) {
 
+    push @extraNixEnvArgs, "--option", "binary-caches", $binaryCacheURL;
 
-print "\nPulling manifests...\n";
-system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0
-    or barf "nix-pull failed: $?";
+} else {
+
+    # Store the manifest in the temporary directory so that we don't
+    # pollute /nix/var/nix/manifests.  This also requires that we
+    # don't use the Nix daemon (because otherwise
+    # download-using-manifests won't see our NIX_MANIFESTS_DIRS
+    # environment variable).
+    $ENV{NIX_MANIFESTS_DIR} = $tmpDir;
+    $ENV{NIX_REMOTE} = "";
+
+    print "\nPulling manifests...\n";
+    system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0
+        or barf "nix-pull failed: $?";
+
+}
 
 
 print "\nInstalling package...\n";

scripts/nix-push.in

@@ -1,77 +1,77 @@
 #! @perl@ -w @perlFlags@
 
 use strict;
+use File::Basename;
 use File::Temp qw(tempdir);
+use File::Path qw(mkpath);
 use File::stat;
+use File::Copy;
 use Nix::Config;
+use Nix::Store;
 use Nix::Manifest;
 
-my $hashAlgo = "sha256";
-
 my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
 my $nixExpr = "$tmpDir/create-nars.nix";
-my $manifest = "$tmpDir/MANIFEST";
-
-my $curl = "$Nix::Config::curl --fail --silent";
-my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
-$curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
 
 # Parse the command line.
-my $localCopy;
-my $localArchivesDir;
-my $localManifestFile;
-
-my $targetArchivesUrl;
-
-my $archivesPutURL;
-my $archivesGetURL;
-my $manifestPutURL;
+my $compressionType = "xz";
+my $force = 0;
+my $destDir;
+my $writeManifest = 0;
+my $archivesURL;
+my @roots;
 
 sub showSyntax {
     print STDERR <<EOF
-Usage: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...
-   or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL MANIFEST_PUT_URL PATHS...
+Usage: nix-push --dest DIR [--manifest] [--url-prefix URL] PATHS...
 
-`nix-push' copies or uploads the closure of PATHS to the given
-destination.
+`nix-push' packs the closure of PATHS into a set of NAR files stored
+in DIR.  Optionally generate a manifest.
 EOF
     ; # `
     exit 1;
 }
 
-showSyntax if scalar @ARGV < 1;
+for (my $n = 0; $n < scalar @ARGV; $n++) {
+    my $arg = $ARGV[$n];
 
-if ($ARGV[0] eq "--copy") {
-    showSyntax if scalar @ARGV < 3;
-    $localCopy = 1;
-    shift @ARGV;
-    $localArchivesDir = shift @ARGV;
-    $localManifestFile = shift @ARGV;
-    if ($ARGV[0] eq "--target") {
-       shift @ARGV;
-       $targetArchivesUrl = shift @ARGV;
-    }
-    else {
-       $targetArchivesUrl = "file://$localArchivesDir";
+    if ($arg eq "--help") {
+        showSyntax;
+    } elsif ($arg eq "--bzip2") {
+        $compressionType = "bzip2";
+    } elsif ($arg eq "--force") {
+        $force = 1;
+    } elsif ($arg eq "--dest") {
+        $n++;
+        die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
+        $destDir = $ARGV[$n];
+        mkpath($destDir, 0, 0755);
+    } elsif ($arg eq "--manifest") {
+        $writeManifest = 1;
+    } elsif ($arg eq "--url-prefix") {
+        $n++;
+        die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
+        $archivesURL = $ARGV[$n];
+    } elsif (substr($arg, 0, 1) eq "-") {
+        showSyntax;
+    } else {
+        push @roots, $arg;
     }
 }
-else {
-    showSyntax if scalar @ARGV < 3;
-    $localCopy = 0;
-    $archivesPutURL = shift @ARGV;
-    $archivesGetURL = shift @ARGV;
-    $manifestPutURL = shift @ARGV;
-}
+
+showSyntax if !defined $destDir;
+
+$archivesURL = "file://$destDir" unless defined $archivesURL;
 
 
 # From the given store paths, determine the set of requisite store
 # paths, i.e, the paths required to realise them.
 my %storePaths;
 
-foreach my $path (@ARGV) {
+foreach my $path (@roots) {
     die unless $path =~ /^\//;
 
     # Get all paths referenced by the normalisation of the given
@@ -92,8 +92,8 @@ foreach my $path (@ARGV) {
 my @storePaths = keys %storePaths;
 
 
-# For each path, create a Nix expression that turns the path into
-# a Nix archive.
+# Create a list of Nix derivations that turn each path into a Nix
+# archive.
 open NIX, ">$nixExpr";
 print NIX "[";
 
@@ -103,7 +103,7 @@ foreach my $storePath (@storePaths) {
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr =
         "(import <nix/nar.nix> " .
-        "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"$hashAlgo\"; }) ";
+        "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
 
     print NIX $nixexpr;
 }
@@ -112,172 +112,132 @@ print NIX "]";
 close NIX;
 
 
-# Instantiate store derivations from the Nix expression.
-my @storeExprs;
-print STDERR "instantiating store derivations...\n";
-my $pid = open(READ, "$Nix::Config::binDir/nix-instantiate $nixExpr|")
-    or die "cannot run nix-instantiate";
+# Build the Nix expression.
+print STDERR "building compressed archives...\n";
+my @narPaths;
+my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |")
+    or die "cannot run nix-build";
 while (<READ>) {
     chomp;
     die unless /^\//;
-    push @storeExprs, $_;
+    push @narPaths, $_;
 }
-close READ or die "nix-instantiate failed: $?";
+close READ or die "nix-build failed: $?";
 
 
-# Build the derivations.
-print STDERR "creating archives...\n";
-
-my @narPaths;
-
-my @tmp = @storeExprs;
-while (scalar @tmp > 0) {
-    my $n = scalar @tmp;
-    if ($n > 256) { $n = 256 };
-    my @tmp2 = @tmp[0..$n - 1];
-    @tmp = @tmp[$n..scalar @tmp - 1];
-
-    my $pid = open(READ, "$Nix::Config::binDir/nix-store --realise @tmp2|")
-        or die "cannot run nix-store";
-    while (<READ>) {
-        chomp;
-        die unless (/^\//);
-        push @narPaths, "$_";
-    }
-    close READ or die "nix-store failed: $?";
+# Write the cache info file.
+my $cacheInfoFile = "$destDir/nix-cache-info";
+if (! -e $cacheInfoFile) {
+    open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
+    print FILE "StoreDir: $Nix::Config::storeDir\n";
+    print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
+    close FILE;
 }
 
 
-# Create the manifest.
-print STDERR "creating manifest...\n";
+# Copy the archives and the corresponding NAR info files.
+print STDERR "copying archives...\n";
+
+my $totalNarSize = 0;
+my $totalCompressedSize = 0;
 
 my %narFiles;
-my %patches;
 
-my @narArchives;
 for (my $n = 0; $n < scalar @storePaths; $n++) {
     my $storePath = $storePaths[$n];
     my $narDir = $narPaths[$n];
+    my $baseName = basename $storePath;
 
-    $storePath =~ /\/([^\/]*)$/;
-    my $basename = $1;
-    defined $basename or die;
-
-    open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
-    my $narbz2Hash = <HASH>;
-    chomp $narbz2Hash;
-    $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
-    close HASH;
-
-    my $narName = "$narbz2Hash.nar.bz2";
-
-    my $narFile = "$narDir/$narName";
-    (-f $narFile) or die "narfile for $storePath not found";
-    push @narArchives, $narFile;
-
-    my $narbz2Size = stat($narFile)->size;
-
-    my $references = `$Nix::Config::binDir/nix-store --query --references '$storePath'`;
-    die "cannot query references for `$storePath'" if $? != 0;
-    $references = join(" ", split(" ", $references));
-
-    my $deriver = `$Nix::Config::binDir/nix-store --query --deriver '$storePath'`;
-    die "cannot query deriver for `$storePath'" if $? != 0;
-    chomp $deriver;
-    $deriver = "" if $deriver eq "unknown-deriver";
-
-    my $narHash = `$Nix::Config::binDir/nix-store --query --hash '$storePath'`;
-    die "cannot query hash for `$storePath'" if $? != 0;
-    chomp $narHash;
+    # Get info about the store path.
+    my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
 
     # In some exceptional cases (such as VM tests that use the Nix
     # store of the host), the database doesn't contain the hash.  So
     # compute it.
     if ($narHash =~ /^sha256:0*$/) {
-        $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 '$storePath'`;
-        die "cannot hash `$storePath'" if $? != 0;
+        my $nar = "$tmpDir/nar";
+        system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
+            or die "cannot dump $storePath\n";
+        $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`;
+        die "cannot hash `$nar'" if $? != 0;
         chomp $narHash;
         $narHash = "sha256:$narHash";
+        $narSize = stat("$nar")->size;
+        unlink $nar or die;
     }
 
-    my $narSize = `$Nix::Config::binDir/nix-store --query --size '$storePath'`;
-    die "cannot query size for `$storePath'" if $? != 0;
-    chomp $narSize;
+    $totalNarSize += $narSize;
 
-    my $url;
-    if ($localCopy) {
-        $url = "$targetArchivesUrl/$narName";
-    } else {
-        $url = "$archivesGetURL/$narName";
+    # Get info about the compressed NAR.
+    open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
+    my $compressedHash = <HASH>;
+    chomp $compressedHash;
+    $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
+
+    my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
+
+    my $narFile = "$narDir/$narName";
+    (-f $narFile) or die "NAR file for $storePath not found";
+
+    my $compressedSize = stat($narFile)->size;
+    $totalCompressedSize += $compressedSize;
+
+    printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
+        $compressedSize / (1024 * 1024), $compressedSize / $narSize * 100;
+
+    # Copy the compressed NAR.
+    my $dst = "$destDir/$narName";
+    if (! -f $dst) {
+        my $tmp = "$destDir/.tmp.$$.$narName";
+        copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
+        rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
     }
+
+    # Write the info file.
+    my $info;
+    $info .= "StorePath: $storePath\n";
+    $info .= "URL: $narName\n";
+    $info .= "Compression: $compressionType\n";
+    $info .= "FileHash: sha256:$compressedHash\n";
+    $info .= "FileSize: $compressedSize\n";
+    $info .= "NarHash: $narHash\n";
+    $info .= "NarSize: $narSize\n";
+    $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
+    if (defined $deriver) {
+        $info .= "Deriver: " . basename $deriver . "\n";
+        if (isValidPath($deriver)) {
+            my $drv = derivationFromPath($deriver);
+            $info .= "System: $drv->{platform}\n";
+        }
+    }
+
+    my $pathHash = substr(basename($storePath), 0, 32);
+
+    $dst = "$destDir/$pathHash.narinfo";
+    if ($force || ! -f $dst) {
+        my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
+        open INFO, ">$tmp" or die;
+        print INFO "$info" or die;
+        close INFO or die;
+        rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
+    }
+
     $narFiles{$storePath} = [
-        { url => $url
-        , hash => "$hashAlgo:$narbz2Hash"
-        , size => $narbz2Size
+        { url => "$archivesURL/$narName"
+        , hash => "sha256:$compressedHash"
+        , size => $compressedSize
         , narHash => "$narHash"
         , narSize => $narSize
-        , references => $references
+        , references => join(" ", @{$refs})
         , deriver => $deriver
         }
-    ];
+    ] if $writeManifest;
 }
 
-writeManifest $manifest, \%narFiles, \%patches;
+printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
+    $totalCompressedSize / (1024 * 1024), $totalCompressedSize / $totalNarSize * 100;
 
 
-sub copyFile {
-    my $src = shift;
-    my $dst = shift;
-    my $tmp = "$dst.tmp.$$";
-    system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file";
-    rename($tmp, $dst) or die "cannot rename file: $!";
-}
-
-
-# Upload/copy the archives.
-print STDERR "uploading/copying archives...\n";
-
-sub archiveExists {
-    my $name = shift;
-    print STDERR "  HEAD on $archivesGetURL/$name\n";
-    return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
-}
-
-foreach my $narArchive (@narArchives) {
-
-    $narArchive =~ /\/([^\/]*)$/;
-    my $basename = $1;
-
-    if ($localCopy) {
-        # Since nix-push creates $dst atomically, if it exists we
-        # don't have to copy again.
-        my $dst = "$localArchivesDir/$basename";
-        if (! -f "$localArchivesDir/$basename") {
-            print STDERR "  $narArchive\n";
-            copyFile $narArchive, $dst;
-        }
-    }
-    else {
-        if (!archiveExists("$basename")) {
-            print STDERR "  $narArchive\n";
-            system("$curl --show-error --upload-file " .
-                   "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
-                   die "curl failed on $narArchive: $?";
-        }
-    }
-}
-
-
-# Upload the manifest.
-print STDERR "uploading manifest...\n";
-if ($localCopy) {
-    copyFile $manifest, $localManifestFile;
-    copyFile "$manifest.bz2", "$localManifestFile.bz2";
-} else {
-    system("$curl --show-error --upload-file " .
-           "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
-           die "curl failed on $manifest: $?";
-    system("$curl --show-error --upload-file " .
-           "'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or
-           die "curl failed on $manifest: $?";
-}
+# Optionally write a manifest.
+writeManifest "$destDir/MANIFEST", \%narFiles, \() if $writeManifest;

src/libexpr/eval.cc

@@ -181,7 +181,7 @@ EvalState::EvalState()
     searchPathInsertionPoint = searchPath.end();
     Strings paths = tokenizeString(getEnv("NIX_PATH", ""), ":");
     foreach (Strings::iterator, i, paths) addToSearchPath(*i);
-    addToSearchPath("nix=" + nixDataDir + "/nix/corepkgs");
+    addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
     searchPathInsertionPoint = searchPath.begin();
 
     createBaseEnv();
@@ -1091,7 +1091,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
         if (srcToStore[path] != "")
             dstPath = srcToStore[path];
         else {
-            dstPath = readOnlyMode
+            dstPath = settings.readOnlyMode
                 ? computeStorePathForPath(path).first
                 : store->addToStore(path);
             srcToStore[path] = dstPath;

src/libexpr/get-drvs.hh

@@ -32,6 +32,8 @@ private:
     bool metaInfoRead;
     MetaInfo meta;
 
+    bool failed; // set if we get an AssertionError
+    
 public:
     string name;
     string attrPath; /* path towards the derivation */
@@ -40,7 +42,7 @@ public:
     /* !!! make this private */
     Bindings * attrs;
 
-    DrvInfo() : metaInfoRead(false), attrs(0) { };
+    DrvInfo() : metaInfoRead(false), failed(false), attrs(0) { };
 
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
@@ -58,6 +60,9 @@ public:
     }
 
     void setMetaInfo(const MetaInfo & meta);
+
+    void setFailed() { failed = true; };
+    bool hasFailed() { return failed; };
 };
 
 

src/libexpr/primops.cc

@@ -51,6 +51,12 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
                 % path % ctx);
         if (isDerivation(ctx))
             try {
+                /* For performance, prefetch all substitute info. */
+                PathSet willBuild, willSubstitute, unknown;
+                unsigned long long downloadSize, narSize;
+                queryMissing(*store, singleton<PathSet>(ctx),
+                    willBuild, willSubstitute, unknown, downloadSize, narSize);
+                  
                 /* !!! If using a substitute, we only need to fetch
                    the selected output of this derivation. */
                 store->buildPaths(singleton<PathSet>(ctx));
@@ -617,7 +623,7 @@ static void prim_toFile(EvalState & state, Value * * args, Value & v)
         refs.insert(path);
     }
     
-    Path storePath = readOnlyMode
+    Path storePath = settings.readOnlyMode
         ? computeStorePathForText(name, contents, refs)
         : store->addTextToStore(name, contents, refs);
 
@@ -681,7 +687,7 @@ static void prim_filterSource(EvalState & state, Value * * args, Value & v)
 
     FilterFromExpr filter(state, *args[0]);
 
-    Path dstPath = readOnlyMode
+    Path dstPath = settings.readOnlyMode
         ? computeStorePathForPath(path, true, htSHA256, filter).first
         : store->addToStore(path, true, htSHA256, filter);
 
@@ -1134,7 +1140,7 @@ void EvalState::createBaseEnv()
     mkInt(v, time(0));
     addConstant("__currentTime", v);
 
-    mkString(v, thisSystem.c_str());
+    mkString(v, settings.thisSystem.c_str());
     addConstant("__currentSystem", v);
 
     // Miscellaneous

src/libmain/shared.cc

@@ -64,7 +64,7 @@ void printMissing(StoreAPI & store, const PathSet & paths)
 
     if (!unknown.empty()) {
         printMsg(lvlInfo, format("don't know how to build these paths%1%:")
-            % (readOnlyMode ? " (may be caused by read-only store access)" : ""));
+            % (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""));
         foreach (PathSet::iterator, i, unknown)
             printMsg(lvlInfo, format("  %1%") % *i);
     }
@@ -83,11 +83,20 @@ static void setLogType(string lt)
 static bool showTrace = false;
 
 
+string getArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end)
+{
+    ++i;
+    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
+    return *i;
+}
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
 {
-    setDefaultsFromEnvironment();
+    settings.processEnvironment();
+    settings.loadConfFile();
 
     /* Catch SIGINT. */
     struct sigaction act;
@@ -146,20 +155,19 @@ static void initAndRun(int argc, char * * argv)
     remaining.clear();
 
     /* Process default options. */
-    int verbosityDelta = 0;
+    int verbosityDelta = lvlInfo;
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
         if (arg == "--verbose" || arg == "-v") verbosityDelta++;
         else if (arg == "--quiet") verbosityDelta--;
         else if (arg == "--log-type") {
-            ++i;
-            if (i == args.end()) throw UsageError("`--log-type' requires an argument");
-            setLogType(*i);
+            string s = getArg(arg, i, args.end());
+            setLogType(s);
         }
         else if (arg == "--no-build-output" || arg == "-Q")
-            buildVerbosity = lvlVomit;
+            settings.buildVerbosity = lvlVomit;
         else if (arg == "--print-build-trace")
-            printBuildTrace = true;
+            settings.printBuildTrace = true;
         else if (arg == "--help") {
             printHelp();
             return;
@@ -169,23 +177,23 @@ static void initAndRun(int argc, char * * argv)
             return;
         }
         else if (arg == "--keep-failed" || arg == "-K")
-            keepFailed = true;
+            settings.keepFailed = true;
         else if (arg == "--keep-going" || arg == "-k")
-            keepGoing = true;
+            settings.keepGoing = true;
         else if (arg == "--fallback")
-            tryFallback = true;
+            settings.set("build-fallback", "true");
         else if (arg == "--max-jobs" || arg == "-j")
-            maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end());
+            settings.set("build-max-jobs", getArg(arg, i, args.end()));
         else if (arg == "--cores")
-            buildCores = getIntArg<unsigned int>(arg, i, args.end());
+            settings.set("build-cores", getArg(arg, i, args.end()));
         else if (arg == "--readonly-mode")
-            readOnlyMode = true;
+            settings.readOnlyMode = true;
         else if (arg == "--max-silent-time")
-            maxSilentTime = getIntArg<unsigned int>(arg, i, args.end());
+            settings.set("build-max-silent-time", getArg(arg, i, args.end()));
         else if (arg == "--timeout")
-            buildTimeout = getIntArg<unsigned int>(arg, i, args.end());
+            settings.set("build-timeout", getArg(arg, i, args.end()));
         else if (arg == "--no-build-hook")
-            useBuildHook = false;
+            settings.useBuildHook = false;
         else if (arg == "--show-trace")
             showTrace = true;
         else if (arg == "--option") {
@@ -193,14 +201,15 @@ static void initAndRun(int argc, char * * argv)
             string name = *i;
             ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
             string value = *i;
-            overrideSetting(name, tokenizeString(value));
+            settings.set(name, value);
         }
         else remaining.push_back(arg);
     }
 
-    verbosityDelta += queryIntSetting("verbosity", lvlInfo);
     verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
 
+    settings.update();
+
     run(remaining);
 
     /* Close the Nix database. */

src/libstore/build.cc

@@ -94,7 +94,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap;
 class Goal : public boost::enable_shared_from_this<Goal>
 {
 public:
-    typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode;
+    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
 
 protected:
 
@@ -111,6 +111,10 @@ protected:
     /* Number of goals we are/were waiting for that have failed. */
     unsigned int nrFailed;
 
+    /* Number of substitution goals we are/were waiting for that
+       failed because there are no substituters. */
+    unsigned int nrNoSubstituters;
+
     /* Name of this goal for debugging purposes. */
     string name;
 
@@ -119,7 +123,7 @@ protected:
 
     Goal(Worker & worker) : worker(worker)
     {
-        nrFailed = 0;
+        nrFailed = nrNoSubstituters = 0;
         exitCode = ecBusy;
     }
 
@@ -225,8 +229,6 @@ private:
 
 public:
 
-    bool cacheFailure;
-
     /* Set if at least one derivation had a BuildError (i.e. permanent
        failure). */
     bool permanentFailure;
@@ -306,9 +308,11 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
     trace(format("waitee `%1%' done; %2% left") %
         waitee->name % waitees.size());
 
-    if (result == ecFailed) ++nrFailed;
+    if (result == ecFailed || result == ecNoSubstituters) ++nrFailed;
 
-    if (waitees.empty() || (result == ecFailed && !keepGoing)) {
+    if (result == ecNoSubstituters) ++nrNoSubstituters;
+
+    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
 
         /* If we failed and keepGoing is not set, we remove all
            remaining waitees. */
@@ -330,7 +334,7 @@ void Goal::amDone(ExitCode result)
 {
     trace("done");
     assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed);
+    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
     exitCode = result;
     foreach (WeakGoals::iterator, i, waiters) {
         GoalPtr goal = i->lock();
@@ -365,6 +369,7 @@ void commonChildInit(Pipe & logPipe)
     if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
         throw SysError("cannot pipe standard error into log file");
     logPipe.readSide.close();
+    logPipe.writeSide.close();
 
     /* Dup stderr to stdout. */
     if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
@@ -459,14 +464,13 @@ void UserLock::acquire()
 {
     assert(uid == 0);
 
-    string buildUsersGroup = querySetting("build-users-group", "");
-    assert(buildUsersGroup != "");
+    assert(settings.buildUsersGroup != "");
 
     /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(buildUsersGroup.c_str());
+    struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
     if (!gr)
         throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
-            % buildUsersGroup);
+            % settings.buildUsersGroup);
     gid = gr->gr_gid;
 
     /* Copy the result of getgrnam. */
@@ -478,7 +482,7 @@ void UserLock::acquire()
 
     if (users.empty())
         throw Error(format("the build users group `%1%' has no members")
-            % buildUsersGroup);
+            % settings.buildUsersGroup);
 
     /* Find a user account that isn't currently in use for another
        build. */
@@ -488,11 +492,11 @@ void UserLock::acquire()
         struct passwd * pw = getpwnam(i->c_str());
         if (!pw)
             throw Error(format("the user `%1%' in the group `%2%' does not exist")
-                % *i % buildUsersGroup);
+                % *i % settings.buildUsersGroup);
 
-        createDirs(nixStateDir + "/userpool");
+        createDirs(settings.nixStateDir + "/userpool");
 
-        fnUserLock = (format("%1%/userpool/%2%") % nixStateDir % pw->pw_uid).str();
+        fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
 
         if (lockedPaths.find(fnUserLock) != lockedPaths.end())
             /* We already have a lock on this one. */
@@ -512,7 +516,7 @@ void UserLock::acquire()
             /* Sanity check... */
             if (uid == getuid() || uid == geteuid())
                 throw Error(format("the Nix user should not be a member of `%1%'")
-                    % buildUsersGroup);
+                    % settings.buildUsersGroup);
 
             return;
         }
@@ -520,7 +524,7 @@ void UserLock::acquire()
 
     throw Error(format("all build users are currently in use; "
         "consider creating additional users and adding them to the `%1%' group")
-        % buildUsersGroup);
+        % settings.buildUsersGroup);
 }
 
 
@@ -539,7 +543,7 @@ static void runSetuidHelper(const string & command,
     const string & arg)
 {
     Path program = getEnv("NIX_SETUID_HELPER",
-        nixLibexecDir + "/nix-setuid-helper");
+        settings.nixLibexecDir + "/nix-setuid-helper");
 
     /* Fork. */
     Pid pid;
@@ -594,12 +598,6 @@ bool amPrivileged()
 }
 
 
-bool haveBuildUsers()
-{
-    return querySetting("build-users-group", "") != "";
-}
-
-
 void getOwnership(const Path & path)
 {
     runSetuidHelper("get-ownership", path);
@@ -614,7 +612,7 @@ void deletePathWrapped(const Path & path, unsigned long long & bytesFreed)
     } catch (SysError & e) {
         /* If this failed due to a permission error, then try it with
            the setuid helper. */
-        if (haveBuildUsers() && !amPrivileged()) {
+        if (settings.buildUsersGroup != "" && !amPrivileged()) {
             getOwnership(path);
             deletePath(path, bytesFreed);
         } else
@@ -692,10 +690,10 @@ HookInstance::HookInstance()
             if (dup2(builderOut.writeSide, 4) == -1)
                 throw SysError("dupping builder's stdout/stderr");
 
-	    /* XXX: Pass `buildTimeout' to the hook?  */
-            execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(),
-                (format("%1%") % maxSilentTime).str().c_str(),
-                (format("%1%") % printBuildTrace).str().c_str(),
+            /* XXX: Pass `buildTimeout' to the hook?  */
+            execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
+                (format("%1%") % settings.maxSilentTime).str().c_str(),
+                (format("%1%") % settings.printBuildTrace).str().c_str(),
                 NULL);
 
             throw SysError(format("executing `%1%'") % buildHook);
@@ -735,6 +733,8 @@ HookInstance::~HookInstance()
 
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
+class SubstitutionGoal;
+
 class DerivationGoal : public Goal
 {
 private:
@@ -933,7 +933,7 @@ void DerivationGoal::init()
 {
     trace("init");
 
-    if (readOnlyMode)
+    if (settings.readOnlyMode)
         throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
 
     /* The first thing to do is to make sure that the derivation
@@ -985,10 +985,8 @@ void DerivationGoal::haveDerivation()
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    foreach (PathSet::iterator, i, invalidOutputs)
-        /* Don't bother creating a substitution goal if there are no
-           substitutes. */
-        if (queryBoolSetting("build-use-substitutes", true) && worker.store.hasSubstitutes(*i))
+    if (settings.useSubstitutes)
+        foreach (PathSet::iterator, i, invalidOutputs)
             addWaitee(worker.makeSubstitutionGoal(*i));
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -1002,10 +1000,10 @@ void DerivationGoal::outputsSubstituted()
 {
     trace("all outputs substituted (maybe)");
 
-    if (nrFailed > 0 && !tryFallback)
+    if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback)
         throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
 
-    nrFailed = 0;
+    nrFailed = nrNoSubstituters = 0;
 
     if (checkPathValidity(false).size() == 0) {
         amDone(ecSuccess);
@@ -1101,9 +1099,9 @@ PathSet outputPaths(const DerivationOutputs & outputs)
 
 static bool canBuildLocally(const string & platform)
 {
-    return platform == thisSystem 
+    return platform == settings.thisSystem
 #ifdef CAN_DO_LINUX32_BUILDS
-        || (platform == "i686-linux" && thisSystem == "x86_64-linux")
+        || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
 #endif
         ;
 }
@@ -1205,7 +1203,7 @@ void DerivationGoal::tryToBuild()
        derivation prefers to be done locally, do it even if
        maxBuildJobs is 0. */
     unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
+    if (curBuilds >= settings.maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
         worker.waitForBuildSlot(shared_from_this());
         outputLocks.unlock();
         return;
@@ -1220,7 +1218,7 @@ void DerivationGoal::tryToBuild()
         printMsg(lvlError, e.msg());
         outputLocks.unlock();
         buildUser.release();
-        if (printBuildTrace)
+        if (settings.printBuildTrace)
             printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
                 % drvPath % drv.outputs["out"].path % 0 % e.msg());
         worker.permanentFailure = true;
@@ -1352,7 +1350,7 @@ void DerivationGoal::buildDone()
         bool hookError = hook &&
             (!WIFEXITED(status) || WEXITSTATUS(status) != 100);
 
-        if (printBuildTrace) {
+        if (settings.printBuildTrace) {
             if (hook && hookError)
                 printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
                     % drvPath % drv.outputs["out"].path % status % e.msg());
@@ -1368,7 +1366,7 @@ void DerivationGoal::buildDone()
            able to access the network).  Hook errors (like
            communication problems with the remote machine) shouldn't
            be cached either. */
-        if (worker.cacheFailure && !hookError && !fixedOutput)
+        if (settings.cacheFailure && !hookError && !fixedOutput)
             foreach (DerivationOutputs::iterator, i, drv.outputs)
                 worker.store.registerFailedPath(i->second.path);
 
@@ -1380,7 +1378,7 @@ void DerivationGoal::buildDone()
     /* Release the build user, if applicable. */
     buildUser.release();
 
-    if (printBuildTrace) {
+    if (settings.printBuildTrace) {
         printMsg(lvlError, format("@ build-succeeded %1% %2%")
             % drvPath % drv.outputs["out"].path);
     }
@@ -1391,7 +1389,7 @@ void DerivationGoal::buildDone()
 
 HookReply DerivationGoal::tryBuildHook()
 {
-    if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
+    if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
 
     if (!worker.hook)
         worker.hook = boost::shared_ptr<HookInstance>(new HookInstance);
@@ -1404,7 +1402,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Send the request to the hook. */
     writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
-        % (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0")
+        % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
         % drv.platform % drvPath % concatStringsSep(",", features)).str());
 
     /* Read the first line of input, which should be a word indicating
@@ -1463,7 +1461,7 @@ HookReply DerivationGoal::tryBuildHook()
     fds.insert(hook->builderOut.readSide);
     worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
 
-    if (printBuildTrace)
+    if (settings.printBuildTrace)
         printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
             % drvPath % drv.outputs["out"].path % drv.platform % logFile);
 
@@ -1494,7 +1492,7 @@ void DerivationGoal::startBuilder()
     if (!canBuildLocally(drv.platform))
         throw Error(
             format("a `%1%' is required to build `%3%', but I am a `%2%'")
-            % drv.platform % thisSystem % drvPath);
+            % drv.platform % settings.thisSystem % drvPath);
 
     /* Construct the environment passed to the builder. */
 
@@ -1515,10 +1513,10 @@ void DerivationGoal::startBuilder()
        shouldn't care, but this is useful for purity checking (e.g.,
        the compiler or linker might only want to accept paths to files
        in the store or in the build directory). */
-    env["NIX_STORE"] = nixStore;
+    env["NIX_STORE"] = settings.nixStore;
 
     /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str();
+    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
 
     /* Add all bindings specified in the derivation. */
     foreach (StringPairs::iterator, i, drv.env)
@@ -1610,7 +1608,7 @@ void DerivationGoal::startBuilder()
 
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
-    if (haveBuildUsers()) {
+    if (settings.buildUsersGroup != "") {
         buildUser.acquire();
         assert(buildUser.getUID() != 0);
         assert(buildUser.getGID() != 0);
@@ -1632,15 +1630,15 @@ void DerivationGoal::startBuilder()
            the builder can create its output but not mess with the
            outputs of other processes). */
         struct stat st;
-        if (stat(nixStore.c_str(), &st) == -1)
-            throw SysError(format("cannot stat `%1%'") % nixStore);
+        if (stat(settings.nixStore.c_str(), &st) == -1)
+            throw SysError(format("cannot stat `%1%'") % settings.nixStore);
         if (!(st.st_mode & S_ISVTX) ||
             ((st.st_mode & S_IRWXG) != S_IRWXG) ||
             (st.st_gid != buildUser.getGID()))
             throw Error(format(
                 "builder does not have write permission to `%2%'; "
                 "try `chgrp %1% %2%; chmod 1775 %2%'")
-                % buildUser.getGID() % nixStore);
+                % buildUser.getGID() % settings.nixStore);
     }
 
 
@@ -1649,7 +1647,7 @@ void DerivationGoal::startBuilder()
        functions like fetchurl (which needs a proper /etc/resolv.conf)
        work properly.  Purity checking for fixed-output derivations
        is somewhat pointless anyway. */
-    useChroot = queryBoolSetting("build-use-chroot", false);
+    useChroot = settings.useChroot;
 
     if (fixedOutput) useChroot = false;
 
@@ -1689,8 +1687,8 @@ void DerivationGoal::startBuilder()
                 % (buildUser.enabled() ? buildUser.getUID() : getuid())
                 % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
 
-	/* Declare the build user's group so that programs get a consistent
-	   view of the system (e.g., "id -gn"). */
+        /* Declare the build user's group so that programs get a consistent
+           view of the system (e.g., "id -gn"). */
         writeFile(chrootRootDir + "/etc/group",
             (format("nixbld:!:%1%:\n")
                 % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
@@ -1699,16 +1697,8 @@ void DerivationGoal::startBuilder()
         writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
 
         /* Bind-mount a user-configurable set of directories from the
-           host file system.  The `/dev/pts' directory must be mounted
-           separately so that newly-created pseudo-terminals show
-           up. */
-        Paths defaultDirs;
-        defaultDirs.push_back("/dev");
-        defaultDirs.push_back("/dev/pts");
-
-        Paths dirsInChroot_ = querySetting("build-chroot-dirs", defaultDirs);
-        dirsInChroot.insert(dirsInChroot_.begin(), dirsInChroot_.end());
-
+           host file system. */
+        dirsInChroot = settings.dirsInChroot;
         dirsInChroot.insert(tmpDir);
 
         /* Make the closure of the inputs available in the chroot,
@@ -1718,8 +1708,8 @@ void DerivationGoal::startBuilder()
            can be bind-mounted).  !!! As an extra security
            precaution, make the fake Nix store only writable by the
            build user. */
-        createDirs(chrootRootDir + nixStore);
-        chmod_(chrootRootDir + nixStore, 01777);
+        createDirs(chrootRootDir + settings.nixStore);
+        chmod_(chrootRootDir + settings.nixStore, 01777);
 
         foreach (PathSet::iterator, i, inputPaths) {
             struct stat st;
@@ -1819,7 +1809,7 @@ void DerivationGoal::startBuilder()
     worker.childStarted(shared_from_this(), pid,
         singleton<set<int> >(builderOut.readSide), true, true);
 
-    if (printBuildTrace) {
+    if (settings.printBuildTrace) {
         printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
             % drvPath % drv.outputs["out"].path % drv.platform % logFile);
     }
@@ -1917,16 +1907,14 @@ void DerivationGoal::initChild()
 #ifdef CAN_DO_LINUX32_BUILDS
         /* Change the personality to 32-bit if we're doing an
            i686-linux build on an x86_64-linux machine. */
-        if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") {
+        if (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") {
             if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
                 throw SysError("cannot set i686-linux personality");
         }
 
         /* Impersonate a Linux 2.6 machine to get some determinism in
            builds that depend on the kernel version. */
-        if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") &&
-            queryBoolSetting("build-impersonate-linux-26", true))
-        {
+        if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) {
             int cur = personality(0xffffffff);
             if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
         }
@@ -1968,7 +1956,7 @@ void DerivationGoal::initChild()
 
             } else {
                 /* Let the setuid helper take care of it. */
-                program = nixLibexecDir + "/nix-setuid-helper";
+                program = settings.nixLibexecDir + "/nix-setuid-helper";
                 args.push_back(program.c_str());
                 args.push_back("run-builder");
                 user = buildUser.getUser().c_str();
@@ -2078,12 +2066,12 @@ void DerivationGoal::computeClosure()
         }
 
         /* Get rid of all weird permissions. */
-	canonicalisePathMetaData(path);
+        canonicalisePathMetaData(path);
 
-	/* For this output path, find the references to other paths
-	   contained in it.  Compute the SHA-256 NAR hash at the same
-	   time.  The hash is stored in the database so that we can
-	   verify later on whether nobody has messed with the store. */
+        /* For this output path, find the references to other paths
+           contained in it.  Compute the SHA-256 NAR hash at the same
+           time.  The hash is stored in the database so that we can
+           verify later on whether nobody has messed with the store. */
         HashResult hash;
         PathSet references = scanForReferences(path, allPaths, hash);
         contentHashes[path] = hash;
@@ -2136,13 +2124,13 @@ string drvsLogDir = "drvs";
 
 Path DerivationGoal::openLogFile()
 {
-    if (!queryBoolSetting("build-keep-log", true)) return "";
+    if (!settings.keepLog) return "";
 
     /* Create a log file. */
-    Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str();
+    Path dir = (format("%1%/%2%") % settings.nixLogDir % drvsLogDir).str();
     createDirs(dir);
 
-    if (queryBoolSetting("build-compress-log", true)) {
+    if (settings.compressLog) {
 
         Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
         AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
@@ -2189,9 +2177,9 @@ void DerivationGoal::closeLogFile()
 void DerivationGoal::deleteTmpDir(bool force)
 {
     if (tmpDir != "") {
-        if (keepFailed && !force) {
-	    printMsg(lvlError, 
-		format("builder for `%1%' failed; keeping build directory `%2%'")
+        if (settings.keepFailed && !force) {
+            printMsg(lvlError,
+                format("builder for `%1%' failed; keeping build directory `%2%'")
                 % drvPath % tmpDir);
             if (buildUser.enabled() && !amPrivileged())
                 getOwnership(tmpDir);
@@ -2209,7 +2197,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     if ((hook && fd == hook->builderOut.readSide) ||
         (!hook && fd == builderOut.readSide))
     {
-        if (verbosity >= buildVerbosity)
+        if (verbosity >= settings.buildVerbosity)
             writeToStderr((unsigned char *) data.data(), data.size());
         if (bzLogFile) {
             int err;
@@ -2245,13 +2233,13 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid)
 
 bool DerivationGoal::pathFailed(const Path & path)
 {
-    if (!worker.cacheFailure) return false;
+    if (!settings.cacheFailure) return false;
 
     if (!worker.store.hasPathFailed(path)) return false;
 
     printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path);
 
-    if (printBuildTrace)
+    if (settings.printBuildTrace)
         printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
 
     worker.permanentFailure = true;
@@ -2278,10 +2266,16 @@ private:
     /* The current substituter. */
     Path sub;
 
+    /* Whether any substituter can realise this path */
+    bool hasSubstitute;
+
     /* Path info returned by the substituter's query info operation. */
     SubstitutablePathInfo info;
 
-    /* Pipe for the substitute's standard output/error. */
+    /* Pipe for the substituter's standard output. */
+    Pipe outPipe;
+
+    /* Pipe for the substituter's standard error. */
     Pipe logPipe;
 
     /* The process ID of the builder. */
@@ -2319,6 +2313,7 @@ public:
 
 SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker)
     : Goal(worker)
+    , hasSubstitute(false)
 {
     this->storePath = storePath;
     state = &SubstitutionGoal::init;
@@ -2365,10 +2360,10 @@ void SubstitutionGoal::init()
         return;
     }
 
-    if (readOnlyMode)
+    if (settings.readOnlyMode)
         throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
 
-    subs = substituters;
+    subs = settings.substituters;
 
     tryNext();
 }
@@ -2382,17 +2377,23 @@ void SubstitutionGoal::tryNext()
         /* None left.  Terminate this goal and let someone else deal
            with it. */
         debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
-        amDone(ecFailed);
+        /* Hack: don't indicate failure if there were no substituters.
+           In that case the calling derivation should just do a
+           build. */
+        amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
         return;
     }
 
     sub = subs.front();
     subs.pop_front();
 
-    if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
-        tryNext();
-        return;
-    }
+    SubstitutablePathInfos infos;
+    PathSet dummy(singleton<PathSet>(storePath));
+    worker.store.querySubstitutablePathInfos(sub, dummy, infos);
+    SubstitutablePathInfos::iterator k = infos.find(storePath);
+    if (k == infos.end()) { tryNext(); return; }
+    info = k->second;
+    hasSubstitute = true;
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
@@ -2434,7 +2435,7 @@ void SubstitutionGoal::tryToRun()
        is maxBuildJobs == 0 (no local builds allowed), we still allow
        a substituter to run.  This is because substitutions cannot be
        distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= (maxBuildJobs == 0 ? 1 : maxBuildJobs)) {
+    if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) {
         worker.waitForBuildSlot(shared_from_this());
         return;
     }
@@ -2467,6 +2468,7 @@ void SubstitutionGoal::tryToRun()
 
     printMsg(lvlInfo, format("fetching path `%1%'...") % storePath);
 
+    outPipe.create();
     logPipe.create();
 
     /* Remove the (stale) output path if it exists. */
@@ -2483,10 +2485,17 @@ void SubstitutionGoal::tryToRun()
     case 0:
         try { /* child */
 
-            logPipe.readSide.close();
-
             commonChildInit(logPipe);
 
+            if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
+                throw SysError("cannot dup output pipe into stdout");
+            outPipe.readSide.close();
+            outPipe.writeSide.close();
+
+            /* Pass configuration options (including those overriden
+               with --option) to the substituter. */
+            setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
+
             /* Fill in the arguments. */
             Strings args;
             args.push_back(baseNameOf(sub));
@@ -2507,13 +2516,14 @@ void SubstitutionGoal::tryToRun()
     /* parent */
     pid.setSeparatePG(true);
     pid.setKillSignal(SIGTERM);
+    outPipe.writeSide.close();
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(),
         pid, singleton<set<int> >(logPipe.readSide), true, true);
 
     state = &SubstitutionGoal::finished;
 
-    if (printBuildTrace) {
+    if (settings.printBuildTrace) {
         printMsg(lvlError, format("@ substituter-started %1% %2%")
             % storePath % sub);
     }
@@ -2535,9 +2545,12 @@ void SubstitutionGoal::finished()
     /* Close the read side of the logger pipe. */
     logPipe.readSide.close();
 
-    debug(format("substitute for `%1%' finished") % storePath);
+    /* Get the hash info from stdout. */
+    string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : "";
+    outPipe.readSide.close();
 
     /* Check the exit status and the build result. */
+    HashResult hash;
     try {
 
         if (!statusOk(status))
@@ -2547,11 +2560,28 @@ void SubstitutionGoal::finished()
         if (!pathExists(storePath))
             throw SubstError(format("substitute did not produce path `%1%'") % storePath);
 
+        hash = hashPath(htSHA256, storePath);
+
+        /* Verify the expected hash we got from the substituer. */
+        if (expectedHashStr != "") {
+            size_t n = expectedHashStr.find(':');
+            if (n == string::npos)
+                throw Error(format("bad hash from substituter: %1%") % expectedHashStr);
+            HashType hashType = parseHashType(string(expectedHashStr, 0, n));
+            if (hashType == htUnknown)
+                throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr);
+            Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
+            Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, storePath).first;
+            if (expectedHash != actualHash)
+                throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%")
+                    % storePath % printHash(expectedHash) % printHash(actualHash));
+        }
+
     } catch (SubstError & e) {
 
         printMsg(lvlInfo, e.msg());
 
-        if (printBuildTrace) {
+        if (settings.printBuildTrace) {
             printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
                 % storePath % status % e.msg());
         }
@@ -2564,8 +2594,6 @@ void SubstitutionGoal::finished()
 
     canonicalisePathMetaData(storePath);
 
-    HashResult hash = hashPath(htSHA256, storePath);
-    
     worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
 
     ValidPathInfo info2;
@@ -2581,7 +2609,7 @@ void SubstitutionGoal::finished()
     printMsg(lvlChatty,
         format("substitution of path `%1%' succeeded") % storePath);
 
-    if (printBuildTrace) {
+    if (settings.printBuildTrace) {
         printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
     }
 
@@ -2592,7 +2620,7 @@ void SubstitutionGoal::finished()
 void SubstitutionGoal::handleChildOutput(int fd, const string & data)
 {
     assert(fd == logPipe.readSide);
-    if (verbosity >= buildVerbosity)
+    if (verbosity >= settings.buildVerbosity)
         writeToStderr((unsigned char *) data.data(), data.size());
     /* Don't write substitution output to a log file for now.  We
        probably should, though. */
@@ -2620,7 +2648,6 @@ Worker::Worker(LocalStore & store)
     working = true;
     nrLocalBuilds = 0;
     lastWokenUp = 0;
-    cacheFailure = queryBoolSetting("build-cache-failure", false);
     permanentFailure = false;
 }
 
@@ -2685,7 +2712,7 @@ void Worker::removeGoal(GoalPtr goal)
         topGoals.erase(goal);
         /* If a top-level goal failed, then kill all other goals
            (unless keepGoing was set). */
-        if (goal->getExitCode() == Goal::ecFailed && !keepGoing)
+        if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing)
             topGoals.clear();
     }
 
@@ -2757,7 +2784,7 @@ void Worker::childTerminated(pid_t pid, bool wakeSleepers)
 void Worker::waitForBuildSlot(GoalPtr goal)
 {
     debug("wait for build slot");
-    if (getNrLocalBuilds() < maxBuildJobs)
+    if (getNrLocalBuilds() < settings.maxBuildJobs)
         wakeUp(goal); /* we can do it right away */
     else
         wantingToBuild.insert(goal);
@@ -2806,7 +2833,7 @@ void Worker::run(const Goals & _topGoals)
         if (!children.empty() || !waitingForAWhile.empty())
             waitForInput();
         else {
-            if (awake.empty() && maxBuildJobs == 0) throw Error(
+            if (awake.empty() && settings.maxBuildJobs == 0) throw Error(
                 "unable to start any build; either increase `--max-jobs' "
                 "or enable distributed builds");
             assert(!awake.empty());
@@ -2816,9 +2843,9 @@ void Worker::run(const Goals & _topGoals)
     /* If --keep-going is not set, it's possible that the main goal
        exited while some of its subgoals were still active.  But if
        --keep-going *is* set, then they must all be finished now. */
-    assert(!keepGoing || awake.empty());
-    assert(!keepGoing || wantingToBuild.empty());
-    assert(!keepGoing || children.empty());
+    assert(!settings.keepGoing || awake.empty());
+    assert(!settings.keepGoing || wantingToBuild.empty());
+    assert(!settings.keepGoing || children.empty());
 }
 
 
@@ -2838,15 +2865,15 @@ void Worker::waitForInput()
     time_t before = time(0);
 
     /* If a global timeout has been set, sleep until it's done.  */
-    if (buildTimeout != 0) {
-	useTimeout = true;
+    if (settings.buildTimeout != 0) {
+        useTimeout = true;
         if (lastWait == 0 || lastWait > before) lastWait = before;
-	timeout.tv_sec = std::max((time_t) 0, lastWait + buildTimeout - before);
+        timeout.tv_sec = std::max((time_t) 0, lastWait + settings.buildTimeout - before);
     }
 
     /* If we're monitoring for silence on stdout/stderr, sleep until
        the first deadline for any child. */
-    if (maxSilentTime != 0) {
+    if (settings.maxSilentTime != 0) {
         time_t oldest = 0;
         foreach (Children::iterator, i, children) {
             if (i->second.monitorForSilence) {
@@ -2855,10 +2882,10 @@ void Worker::waitForInput()
             }
         }
         if (oldest) {
-	    time_t silenceTimeout = std::max((time_t) 0, oldest + maxSilentTime - before);
+            time_t silenceTimeout = std::max((time_t) 0, oldest + settings.maxSilentTime - before);
             timeout.tv_sec = useTimeout
-		? std::min(silenceTimeout, timeout.tv_sec)
-		: silenceTimeout;
+                ? std::min(silenceTimeout, timeout.tv_sec)
+                : silenceTimeout;
             useTimeout = true;
             printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
         }
@@ -2866,14 +2893,12 @@ void Worker::waitForInput()
 
     /* If we are polling goals that are waiting for a lock, then wake
        up after a few seconds at most. */
-    int wakeUpInterval = queryIntSetting("build-poll-interval", 5);
-        
     if (!waitingForAWhile.empty()) {
         useTimeout = true;
         if (lastWokenUp == 0)
             printMsg(lvlError, "waiting for locks or build slots...");
         if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
-        timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before);
+        timeout.tv_sec = std::max((time_t) 0, (time_t) (lastWokenUp + settings.pollInterval - before));
     } else lastWokenUp = 0;
 
     using namespace std;
@@ -2939,27 +2964,27 @@ void Worker::waitForInput()
             }
         }
 
-        if (maxSilentTime != 0 &&
+        if (settings.maxSilentTime != 0 &&
             j->second.monitorForSilence &&
-            after - j->second.lastOutput >= (time_t) maxSilentTime)
+            after - j->second.lastOutput >= (time_t) settings.maxSilentTime)
         {
             printMsg(lvlError,
                 format("%1% timed out after %2% seconds of silence")
-                % goal->getName() % maxSilentTime);
+                % goal->getName() % settings.maxSilentTime);
             goal->cancel();
         }
 
-	if (buildTimeout != 0 &&
-	    after - before >= (time_t) buildTimeout)
+        if (settings.buildTimeout != 0 &&
+            after - before >= (time_t) settings.buildTimeout)
         {
             printMsg(lvlError,
                 format("%1% timed out after %2% seconds of activity")
-                % goal->getName() % buildTimeout);
+                % goal->getName() % settings.buildTimeout);
             goal->cancel();
         }
     }
 
-    if (!waitingForAWhile.empty() && lastWokenUp + wakeUpInterval <= after) {
+    if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) {
         lastWokenUp = after;
         foreach (WeakGoals::iterator, i, waitingForAWhile) {
             GoalPtr goal = i->lock();

src/libstore/derivations.cc

@@ -38,7 +38,7 @@ Path writeDerivation(StoreAPI & store,
        held during a garbage collection). */
     string suffix = name + drvExtension;
     string contents = unparseDerivation(drv);
-    return readOnlyMode
+    return settings.readOnlyMode
         ? computeStorePathForText(suffix, contents, references)
         : store.addTextToStore(suffix, contents, references);
 }

src/libstore/gc.cc

@@ -34,7 +34,7 @@ static const int defaultGcLevel = 1000;
 int LocalStore::openGCLock(LockType lockType)
 {
     Path fnGCLock = (format("%1%/%2%")
-        % nixStateDir % gcLockName).str();
+        % settings.nixStateDir % gcLockName).str();
 
     debug(format("acquiring global GC lock `%1%'") % fnGCLock);
 
@@ -85,7 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path)
 {
     string hash = printHash32(hashString(htSHA1, path));
     Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
-        % nixStateDir % gcRootsDir % hash).str());
+        % settings.nixStateDir % gcRootsDir % hash).str());
     createSymlink(realRoot, path);
 }
 
@@ -113,7 +113,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
 
     else {
         if (!allowOutsideRootsDir) {
-            Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+            Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
 
             if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
                 throw Error(format(
@@ -130,7 +130,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
        Instead of reading all the roots, it would be more efficient to
        check if the root is in a directory in or linked from the
        gcroots directory. */
-    if (queryBoolSetting("gc-check-reachability", false)) {
+    if (settings.checkRootReachability) {
         Roots roots = store.findRoots();
         if (roots.find(gcRoot) == roots.end())
             printMsg(lvlError,
@@ -160,7 +160,7 @@ void LocalStore::addTempRoot(const Path & path)
     if (fdTempRoots == -1) {
 
         while (1) {
-            Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str();
+            Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str();
             createDirs(dir);
 
             fnTempRoots = (format("%1%/%2%")
@@ -173,7 +173,7 @@ void LocalStore::addTempRoot(const Path & path)
                    processes with the same pid. */
                 unlink(fnTempRoots.c_str());
 
-	    fdTempRoots = openLockFile(fnTempRoots, true);
+            fdTempRoots = openLockFile(fnTempRoots, true);
 
             fdGCLock.close();
 
@@ -238,10 +238,10 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
     /* Read the `temproots' directory for per-process temporary root
        files. */
     Strings tempRootFiles = readDirectory(
-        (format("%1%/%2%") % nixStateDir % tempRootsDir).str());
+        (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str());
 
     foreach (Strings::iterator, i, tempRootFiles) {
-        Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
+        Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str();
 
         debug(format("reading temporary root file `%1%'") % path);
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
@@ -350,7 +350,7 @@ static void findRoots(StoreAPI & store, const Path & path,
 static Roots findRoots(StoreAPI & store, bool deleteStale)
 {
     Roots roots;
-    Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+    Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
     findRoots(store, rootsDir, true, deleteStale, roots);
     return roots;
 }
@@ -365,7 +365,7 @@ Roots LocalStore::findRoots()
 static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
 {
     Path rootFinder = getEnv("NIX_ROOT_FINDER",
-        nixLibexecDir + "/nix/find-runtime-roots.pl");
+        settings.nixLibexecDir + "/nix/find-runtime-roots.pl");
 
     if (rootFinder.empty()) return;
 
@@ -623,8 +623,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     GCState state(results);
     state.options = options;
 
-    state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false);
-    state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true);
+    state.gcKeepOutputs = settings.gcKeepOutputs;
+    state.gcKeepDerivations = settings.gcKeepDerivations;
 
     /* Using `--ignore-liveness' with `--delete' can have unintended
        consequences if `gc-keep-outputs' or `gc-keep-derivations' are
@@ -685,8 +685,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         try {
 
-            AutoCloseDir dir = opendir(nixStore.c_str());
-            if (!dir) throw SysError(format("opening directory `%1%'") % nixStore);
+            AutoCloseDir dir = opendir(settings.nixStore.c_str());
+            if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore);
 
             /* Read the store and immediately delete all paths that
                aren't valid.  When using --max-freed etc., deleting
@@ -700,14 +700,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 checkInterrupt();
                 string name = dirent->d_name;
                 if (name == "." || name == "..") continue;
-                Path path = nixStore + "/" + name;
+                Path path = settings.nixStore + "/" + name;
                 if (isValidPath(path))
                     entries.push_back(path);
                 else
                     tryToDelete(state, path);
             }
 
-	    dir.close();
+            dir.close();
 
             /* Now delete the unreachable valid paths.  Randomise the
                order in which we delete entries to make the collector

src/libstore/globals.cc

@@ -10,36 +10,63 @@
 namespace nix {
 
 
-string nixStore = "/UNINIT";
-string nixDataDir = "/UNINIT";
-string nixLogDir = "/UNINIT";
-string nixStateDir = "/UNINIT";
-string nixDBPath = "/UNINIT";
-string nixConfDir = "/UNINIT";
-string nixLibexecDir = "/UNINIT";
-string nixBinDir = "/UNINIT";
-
-bool keepFailed = false;
-bool keepGoing = false;
-bool tryFallback = false;
-Verbosity buildVerbosity = lvlError;
-unsigned int maxBuildJobs = 1;
-unsigned int buildCores = 1;
-bool readOnlyMode = false;
-string thisSystem = "unset";
-time_t maxSilentTime = 0;
-time_t buildTimeout = 0;
-Paths substituters;
-bool useBuildHook = true;
-bool printBuildTrace = false;
+Settings settings;
 
 
-static bool settingsRead = false;
+Settings::Settings()
+{
+    keepFailed = false;
+    keepGoing = false;
+    tryFallback = false;
+    buildVerbosity = lvlError;
+    maxBuildJobs = 1;
+    buildCores = 1;
+    readOnlyMode = false;
+    thisSystem = SYSTEM;
+    maxSilentTime = 0;
+    buildTimeout = 0;
+    useBuildHook = true;
+    printBuildTrace = false;
+    reservedSize = 1024 * 1024;
+    fsyncMetadata = true;
+    useSQLiteWAL = true;
+    syncBeforeRegistering = false;
+    useSubstitutes = true;
+    useChroot = false;
+    dirsInChroot.insert("/dev");
+    dirsInChroot.insert("/dev/pts");
+    impersonateLinux26 = false;
+    keepLog = true;
+    compressLog = true;
+    cacheFailure = false;
+    pollInterval = 5;
+    checkRootReachability = false;
+    gcKeepOutputs = false;
+    gcKeepDerivations = true;
+    autoOptimiseStore = true;
+    envKeepDerivations = false;
+}
 
-static std::map<string, Strings> settings;
 
-/* Overriden settings. */
-std::map<string, Strings> settingsCmdline;
+void Settings::processEnvironment()
+{
+    nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
+    nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
+    nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
+    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
+    nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
+    nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
+    nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
+    nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
+
+    string subs = getEnv("NIX_SUBSTITUTERS", "default");
+    if (subs == "default") {
+        substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+        substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+        substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
+    } else
+        substituters = tokenizeString(subs, ":");
+}
 
 
 string & at(Strings & ss, unsigned int n)
@@ -50,7 +77,7 @@ string & at(Strings & ss, unsigned int n)
 }
 
 
-static void readSettings()
+void Settings::loadConfFile()
 {
     Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
     if (!pathExists(settingsFile)) return;
@@ -78,94 +105,102 @@ static void readSettings()
 
         Strings::iterator i = tokens.begin();
         advance(i, 2);
-        settings[name] = Strings(i, tokens.end());
+        settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow
     };
-
-    settings.insert(settingsCmdline.begin(), settingsCmdline.end());
-    
-    settingsRead = true;
 }
 
 
-Strings querySetting(const string & name, const Strings & def)
+void Settings::set(const string & name, const string & value)
 {
-    if (!settingsRead) readSettings();
-    std::map<string, Strings>::iterator i = settings.find(name);
-    return i == settings.end() ? def : i->second;
+    settings[name] = value;
+    overrides[name] = value;
 }
 
 
-string querySetting(const string & name, const string & def)
+void Settings::update()
 {
-    Strings defs;
-    defs.push_back(def);
-
-    Strings value = querySetting(name, defs);
-    if (value.size() != 1)
-        throw Error(format("configuration option `%1%' should not be a list") % name);
-
-    return value.front();
+    get(tryFallback, "build-fallback");
+    get(maxBuildJobs, "build-max-jobs");
+    get(buildCores, "build-cores");
+    get(thisSystem, "system");
+    get(maxSilentTime, "build-max-silent-time");
+    get(buildTimeout, "build-timeout");
+    get(reservedSize, "gc-reserved-space");
+    get(fsyncMetadata, "fsync-metadata");
+    get(useSQLiteWAL, "use-sqlite-wal");
+    get(syncBeforeRegistering, "sync-before-registering");
+    get(useSubstitutes, "build-use-substitutes");
+    get(buildUsersGroup, "build-users-group");
+    get(useChroot, "build-use-chroot");
+    get(dirsInChroot, "build-chroot-dirs");
+    get(impersonateLinux26, "build-impersonate-linux-26");
+    get(keepLog, "build-keep-log");
+    get(compressLog, "build-compress-log");
+    get(cacheFailure, "build-cache-failure");
+    get(pollInterval, "build-poll-interval");
+    get(checkRootReachability, "gc-check-reachability");
+    get(gcKeepOutputs, "gc-keep-outputs");
+    get(gcKeepDerivations, "gc-keep-derivations");
+    get(autoOptimiseStore, "auto-optimise-store");
+    get(envKeepDerivations, "env-keep-derivations");
 }
 
 
-bool queryBoolSetting(const string & name, bool def)
+void Settings::get(string & res, const string & name)
 {
-    string v = querySetting(name, def ? "true" : "false");
-    if (v == "true") return true;
-    else if (v == "false") return false;
+    SettingsMap::iterator i = settings.find(name);
+    if (i == settings.end()) return;
+    res = i->second;
+}
+
+
+void Settings::get(bool & res, const string & name)
+{
+    SettingsMap::iterator i = settings.find(name);
+    if (i == settings.end()) return;
+    if (i->second == "true") res = true;
+    else if (i->second == "false") res = false;
     else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'")
-        % name % v);
+        % name % i->second);
 }
 
 
-unsigned int queryIntSetting(const string & name, unsigned int def)
+void Settings::get(PathSet & res, const string & name)
 {
-    int n;
-    if (!string2Int(querySetting(name, int2String(def)), n) || n < 0)
+    SettingsMap::iterator i = settings.find(name);
+    if (i == settings.end()) return;
+    res.clear();
+    Strings ss = tokenizeString(i->second);
+    res.insert(ss.begin(), ss.end());
+}
+
+
+template<class N> void Settings::get(N & res, const string & name)
+{
+    SettingsMap::iterator i = settings.find(name);
+    if (i == settings.end()) return;
+    if (!string2Int(i->second, res))
         throw Error(format("configuration setting `%1%' should have an integer value") % name);
-    return n;
 }
 
 
-void overrideSetting(const string & name, const Strings & value)
+string Settings::pack()
 {
-    if (settingsRead) settings[name] = value;
-    settingsCmdline[name] = value;
+    string s;
+    foreach (SettingsMap::iterator, i, settings) {
+        if (i->first.find('\n') != string::npos ||
+            i->first.find('=') != string::npos ||
+            i->second.find('\n') != string::npos)
+            throw Error("illegal option name/value");
+        s += i->first; s += '='; s += i->second; s += '\n';
+    }
+    return s;
 }
 
 
-void reloadSettings()
+Settings::SettingsMap Settings::getOverrides()
 {
-    settingsRead = false;
-    settings.clear();
-}
-
-
-void setDefaultsFromEnvironment()
-{
-    /* Setup Nix paths. */
-    nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
-    nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
-    nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
-    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
-    nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
-    nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
-    nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
-    nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
-
-    string subs = getEnv("NIX_SUBSTITUTERS", "default");
-    if (subs == "default") {
-        substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
-        substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
-    } else
-        substituters = tokenizeString(subs, ":");
-
-    /* Get some settings from the configuration file. */
-    thisSystem = querySetting("system", SYSTEM);
-    maxBuildJobs = queryIntSetting("build-max-jobs", 1);
-    buildCores = queryIntSetting("build-cores", 1);
-    maxSilentTime = queryIntSetting("build-max-silent-time", 0);
-    buildTimeout = queryIntSetting("build-timeout", 0);
+    return overrides;
 }
 
 

src/libstore/globals.hh

@@ -2,118 +2,191 @@
 
 #include "types.hh"
 
+#include <map>
+
 
 namespace nix {
 
 
-/* Path names. */
+struct Settings {
 
-/* nixStore is the directory where we generally store atomic and
-   derived files. */
-extern string nixStore;
+    typedef std::map<string, string> SettingsMap;
 
-extern string nixDataDir; /* !!! fix */
+    Settings();
 
-/* nixLogDir is the directory where we log various operations. */ 
-extern string nixLogDir;
+    void processEnvironment();
 
-/* nixStateDir is the directory where state is stored. */
-extern string nixStateDir;
+    void loadConfFile();
 
-/* nixDBPath is the path name of our Berkeley DB environment. */
-extern string nixDBPath;
+    void set(const string & name, const string & value);
 
-/* nixConfDir is the directory where configuration files are
-   stored. */
-extern string nixConfDir;
+    void update();
 
-/* nixLibexecDir is the directory where internal helper programs are
-   stored. */
-extern string nixLibexecDir;
+    string pack();
 
-/* nixBinDir is the directory where the main programs are stored. */
-extern string nixBinDir;
+    SettingsMap getOverrides();
+
+    /* The directory where we store sources and derived files. */
+    Path nixStore;
+
+    Path nixDataDir; /* !!! fix */
+
+    /* The directory where we log various operations. */
+    Path nixLogDir;
+
+    /* The directory where state is stored. */
+    Path nixStateDir;
+
+    /* The directory where we keep the SQLite database. */
+    Path nixDBPath;
+
+    /* The directory where configuration files are stored. */
+    Path nixConfDir;
+
+    /* The directory where internal helper programs are stored. */
+    Path nixLibexecDir;
+
+    /* The directory where the main programs are stored. */
+    Path nixBinDir;
+
+    /* Whether to keep temporary directories of failed builds. */
+    bool keepFailed;
+
+    /* Whether to keep building subgoals when a sibling (another
+       subgoal of the same goal) fails. */
+    bool keepGoing;
+
+    /* Whether, if we cannot realise the known closure corresponding
+       to a derivation, we should try to normalise the derivation
+       instead. */
+    bool tryFallback;
+
+    /* Verbosity level for build output. */
+    Verbosity buildVerbosity;
+
+    /* Maximum number of parallel build jobs.  0 means unlimited. */
+    unsigned int maxBuildJobs;
+
+    /* Number of CPU cores to utilize in parallel within a build,
+       i.e. by passing this number to Make via '-j'. 0 means that the
+       number of actual CPU cores on the local host ought to be
+       auto-detected. */
+    unsigned int buildCores;
+
+    /* Read-only mode.  Don't copy stuff to the store, don't change
+       the database. */
+    bool readOnlyMode;
+
+    /* The canonical system name, as returned by config.guess. */
+    string thisSystem;
+
+    /* The maximum time in seconds that a builer can go without
+       producing any output on stdout/stderr before it is killed.  0
+       means infinity. */
+    time_t maxSilentTime;
+
+    /* The maximum duration in seconds that a builder can run.  0
+       means infinity.  */
+    time_t buildTimeout;
+
+    /* The substituters.  There are programs that can somehow realise
+       a store path without building, e.g., by downloading it or
+       copying it from a CD. */
+    Paths substituters;
+
+    /* Whether to use build hooks (for distributed builds).  Sometimes
+       users want to disable this from the command-line. */
+    bool useBuildHook;
+
+    /* Whether buildDerivations() should print out lines on stderr in
+       a fixed format to allow its progress to be monitored.  Each
+       line starts with a "@".  The following are defined:
+
+       @ build-started <drvpath> <outpath> <system> <logfile>
+       @ build-failed <drvpath> <outpath> <exitcode> <error text>
+       @ build-succeeded <drvpath> <outpath>
+       @ substituter-started <outpath> <substituter>
+       @ substituter-failed <outpath> <exitcode> <error text>
+       @ substituter-succeeded <outpath>
+
+       Best combined with --no-build-output, otherwise stderr might
+       conceivably contain lines in this format printed by the
+       builders. */
+    bool printBuildTrace;
+
+    /* Amount of reserved space for the garbage collector
+       (/nix/var/nix/db/reserved). */
+    off_t reservedSize;
+
+    /* Whether SQLite should use fsync. */
+    bool fsyncMetadata;
+
+    /* Whether SQLite should use WAL mode. */
+    bool useSQLiteWAL;
+
+    /* Whether to call sync() before registering a path as valid. */
+    bool syncBeforeRegistering;
+
+    /* Whether to use substitutes. */
+    bool useSubstitutes;
+
+    /* The Unix group that contains the build users. */
+    string buildUsersGroup;
+
+    /* Whether to build in chroot. */
+    bool useChroot;
+
+    /* The directories from the host filesystem to be included in the
+       chroot. */
+    PathSet dirsInChroot;
+
+    /* Whether to impersonate a Linux 2.6 machine on newer kernels. */
+    bool impersonateLinux26;
+
+    /* Whether to store build logs. */
+    bool keepLog;
+
+    /* Whether to compress logs. */
+    bool compressLog;
+
+    /* Whether to cache build failures. */
+    bool cacheFailure;
+
+    /* How often (in seconds) to poll for locks. */
+    unsigned int pollInterval;
+
+    /* Whether to check if new GC roots can in fact be found by the
+       garbage collector. */
+    bool checkRootReachability;
+
+    /* Whether the garbage collector should keep outputs of live
+       derivations. */
+    bool gcKeepOutputs;
+
+    /* Whether the garbage collector should keep derivers of live
+       paths. */
+    bool gcKeepDerivations;
+
+    /* Whether to automatically replace files with identical contents
+       with hard links. */
+    bool autoOptimiseStore;
+
+    /* Whether to add derivations as a dependency of user environments
+       (to prevent them from being GCed). */
+    bool envKeepDerivations;
+
+private:
+    SettingsMap settings, overrides;
+
+    void get(string & res, const string & name);
+    void get(bool & res, const string & name);
+    void get(PathSet & res, const string & name);
+    template<class N> void get(N & res, const string & name);
+};
 
 
-/* Misc. global flags. */
-
-/* Whether to keep temporary directories of failed builds. */
-extern bool keepFailed;
-
-/* Whether to keep building subgoals when a sibling (another subgoal
-   of the same goal) fails. */
-extern bool keepGoing;
-
-/* Whether, if we cannot realise the known closure corresponding to a
-   derivation, we should try to normalise the derivation instead. */
-extern bool tryFallback;
-
-/* Verbosity level for build output. */
-extern Verbosity buildVerbosity;
-
-/* Maximum number of parallel build jobs.  0 means unlimited. */
-extern unsigned int maxBuildJobs;
-
-/* Number of CPU cores to utilize in parallel within a build, i.e. by passing
-   this number to Make via '-j'. 0 means that the number of actual CPU cores on
-   the local host ought to be auto-detected. */
-extern unsigned int buildCores;
-
-/* Read-only mode.  Don't copy stuff to the store, don't change the
-   database. */
-extern bool readOnlyMode;
-
-/* The canonical system name, as returned by config.guess. */ 
-extern string thisSystem;
-
-/* The maximum time in seconds that a builer can go without producing
-   any output on stdout/stderr before it is killed.  0 means
-   infinity. */
-extern time_t maxSilentTime;
-
-/* The maximum duration in seconds that a builder can run.  0 means
-   infinity.  */
-extern time_t buildTimeout;
-
-/* The substituters.  There are programs that can somehow realise a
-   store path without building, e.g., by downloading it or copying it
-   from a CD. */
-extern Paths substituters;
-
-/* Whether to use build hooks (for distributed builds).  Sometimes
-   users want to disable this from the command-line. */
-extern bool useBuildHook;
-
-/* Whether buildDerivations() should print out lines on stderr in a
-   fixed format to allow its progress to be monitored.  Each line
-   starts with a "@".  The following are defined:
-
-   @ build-started <drvpath> <outpath> <system> <logfile>
-   @ build-failed <drvpath> <outpath> <exitcode> <error text>
-   @ build-succeeded <drvpath> <outpath>
-   @ substituter-started <outpath> <substituter>
-   @ substituter-failed <outpath> <exitcode> <error text>
-   @ substituter-succeeded <outpath>
-
-   Best combined with --no-build-output, otherwise stderr might
-   conceivably contain lines in this format printed by the builders.
-*/
-extern bool printBuildTrace;
-
-
-Strings querySetting(const string & name, const Strings & def);
-
-string querySetting(const string & name, const string & def);
-
-bool queryBoolSetting(const string & name, bool def);
-
-unsigned int queryIntSetting(const string & name, unsigned int def);
-
-void overrideSetting(const string & name, const Strings & value);
-
-void reloadSettings();
-
-void setDefaultsFromEnvironment();
+// FIXME: don't use a global variable.
+extern Settings settings;
 
 
 }

src/libstore/local-store.cc

@@ -181,7 +181,7 @@ struct SQLiteTxn
 void checkStoreNotSymlink()
 {
     if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
-    Path path = nixStore;
+    Path path = settings.nixStore;
     struct stat st;
     while (path != "/") {
         if (lstat(path.c_str(), &st))
@@ -198,23 +198,21 @@ void checkStoreNotSymlink()
 
 LocalStore::LocalStore(bool reserveSpace)
 {
-    substitutablePathsLoaded = false;
+    schemaPath = settings.nixDBPath + "/schema";
 
-    schemaPath = nixDBPath + "/schema";
-    
-    if (readOnlyMode) {
+    if (settings.readOnlyMode) {
         openDB(false);
         return;
     }
 
     /* Create missing state directories if they don't already exist. */
-    createDirs(nixStore);
-    createDirs(linksDir = nixStore + "/.links");
-    Path profilesDir = nixStateDir + "/profiles";
-    createDirs(nixStateDir + "/profiles");
-    createDirs(nixStateDir + "/temproots");
-    createDirs(nixDBPath);
-    Path gcRootsDir = nixStateDir + "/gcroots";
+    createDirs(settings.nixStore);
+    createDirs(linksDir = settings.nixStore + "/.links");
+    Path profilesDir = settings.nixStateDir + "/profiles";
+    createDirs(settings.nixStateDir + "/profiles");
+    createDirs(settings.nixStateDir + "/temproots");
+    createDirs(settings.nixDBPath);
+    Path gcRootsDir = settings.nixStateDir + "/gcroots";
     if (!pathExists(gcRootsDir)) {
         createDirs(gcRootsDir);
         if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1)
@@ -228,13 +226,12 @@ LocalStore::LocalStore(bool reserveSpace)
        needed, we reserve some dummy space that we can free just
        before doing a garbage collection. */
     try {
-        Path reservedPath = nixDBPath + "/reserved";
+        Path reservedPath = settings.nixDBPath + "/reserved";
         if (reserveSpace) {
-            int reservedSize = queryIntSetting("gc-reserved-space", 1024 * 1024);
             struct stat st;
             if (stat(reservedPath.c_str(), &st) == -1 ||
-                st.st_size != reservedSize)
-                writeFile(reservedPath, string(reservedSize, 'X'));
+                st.st_size != settings.reservedSize)
+                writeFile(reservedPath, string(settings.reservedSize, 'X'));
         }
         else
             deletePath(reservedPath);
@@ -244,11 +241,11 @@ LocalStore::LocalStore(bool reserveSpace)
     /* Acquire the big fat lock in shared mode to make sure that no
        schema upgrade is in progress. */
     try {
-        Path globalLockPath = nixDBPath + "/big-lock";
+        Path globalLockPath = settings.nixDBPath + "/big-lock";
         globalLock = openLockFile(globalLockPath.c_str(), true);
     } catch (SysError & e) {
         if (e.errNo != EACCES) throw;
-        readOnlyMode = true;
+        settings.readOnlyMode = true;
         openDB(false);
         return;
     }
@@ -327,7 +324,7 @@ int LocalStore::getSchema()
 void LocalStore::openDB(bool create)
 {
     /* Open the Nix database. */
-    if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db,
+    if (sqlite3_open_v2((settings.nixDBPath + "/db.sqlite").c_str(), &db.db,
             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
         throw Error("cannot open SQLite database");
 
@@ -344,13 +341,13 @@ void LocalStore::openDB(bool create)
        should be safe enough.  If the user asks for it, don't sync at
        all.  This can cause database corruption if the system
        crashes. */
-    string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off";
+    string syncMode = settings.fsyncMetadata ? "normal" : "off";
     if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
         throwSQLiteError(db, "setting synchronous mode");
 
     /* Set the SQLite journal mode.  WAL mode is fastest, so it's the
        default. */
-    string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate";
+    string mode = settings.useSQLiteWAL ? "wal" : "truncate";
     string prevMode;
     {
         SQLiteStmt stmt;
@@ -423,7 +420,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-	throw SysError(format("getting attributes of path `%1%'") % path);
+        throw SysError(format("getting attributes of path `%1%'") % path);
 
     /* Really make sure that the path is of a supported type.  This
        has already been checked in dumpPath(). */
@@ -478,8 +475,8 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
 
     if (recurse && S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
-	foreach (Strings::iterator, i, names)
-	    canonicalisePathMetaData(path + "/" + *i, true);
+        foreach (Strings::iterator, i, names)
+            canonicalisePathMetaData(path + "/" + *i, true);
     }
 
     makeImmutable(path);
@@ -494,7 +491,7 @@ void canonicalisePathMetaData(const Path & path)
        be a symlink, since we can't change its ownership. */
     struct stat st;
     if (lstat(path.c_str(), &st))
-	throw SysError(format("getting attributes of path `%1%'") % path);
+        throw SysError(format("getting attributes of path `%1%'") % path);
 
     if (st.st_uid != geteuid()) {
         assert(S_ISLNK(st.st_mode));
@@ -756,7 +753,16 @@ bool LocalStore::isValidPath(const Path & path)
 }
 
 
-PathSet LocalStore::queryValidPaths()
+PathSet LocalStore::queryValidPaths(const PathSet & paths)
+{
+    PathSet res;
+    foreach (PathSet::const_iterator, i, paths)
+        if (isValidPath(*i)) res.insert(*i);
+    return res;
+}
+
+
+PathSet LocalStore::queryAllValidPaths()
 {
     SQLiteStmt stmt;
     stmt.create(db, "select path from ValidPaths");
@@ -883,7 +889,7 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
 
     SQLiteTxn txn(db);
 
-    Path prefix = nixStore + "/" + hashPart;
+    Path prefix = settings.nixStore + "/" + hashPart;
 
     SQLiteStmtUse use(stmtQueryPathFromHashPart);
     stmtQueryPathFromHashPart.bind(prefix);
@@ -903,10 +909,11 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
 
     debug(format("starting substituter program `%1%'") % substituter);
 
-    Pipe toPipe, fromPipe;
+    Pipe toPipe, fromPipe, errorPipe;
 
     toPipe.create();
     fromPipe.create();
+    errorPipe.create();
 
     run.pid = fork();
 
@@ -924,12 +931,18 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
                written in Perl (i.e. all of them) fail. */
             unsetenv("DYLD_LIBRARY_PATH");
 
+            /* Pass configuration options (including those overriden
+               with --option) to the substituter. */
+            setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
+
             fromPipe.readSide.close();
             toPipe.writeSide.close();
             if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
                 throw SysError("dupping stdin");
             if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
                 throw SysError("dupping stdout");
+            if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
+                throw SysError("dupping stderr");
             closeMostFDs(set<int>());
             execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
             throw SysError(format("executing `%1%'") % substituter);
@@ -943,6 +956,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
 
     run.to = toPipe.writeSide.borrow();
     run.from = fromPipe.readSide.borrow();
+    run.error = errorPipe.readSide.borrow();
 }
 
 
@@ -955,50 +969,79 @@ template<class T> T getIntLine(int fd)
 }
 
 
-bool LocalStore::hasSubstitutes(const Path & path)
+PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 {
-    foreach (Paths::iterator, i, substituters) {
+    PathSet res;
+    foreach (Paths::iterator, i, settings.substituters) {
+        if (res.size() == paths.size()) break;
         RunningSubstituter & run(runningSubstituters[*i]);
         startSubstituter(*i, run);
-        writeLine(run.to, "have\n" + path);
-        if (getIntLine<int>(run.from)) return true;
+        string s = "have ";
+        foreach (PathSet::const_iterator, j, paths)
+            if (res.find(*j) == res.end()) { s += *j; s += " "; }
+        writeLine(run.to, s);
+        while (true) {
+            /* FIXME: we only read stderr when an error occurs, so
+               substituters should only write (short) messages to
+               stderr when they fail.  I.e. they shouldn't write debug
+               output. */
+            try {
+                Path path = readLine(run.from);
+                if (path == "") break;
+                res.insert(path);
+            } catch (EndOfFile e) {
+                throw Error(format("substituter `%1%' failed: %2%") % *i % chomp(drainFD(run.error)));
+            }
+        }
     }
-
-    return false;
+    return res;
 }
 
 
-bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
-    const Path & path, SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const Path & substituter,
+    PathSet & paths, SubstitutablePathInfos & infos)
 {
     RunningSubstituter & run(runningSubstituters[substituter]);
     startSubstituter(substituter, run);
 
-    writeLine(run.to, "info\n" + path);
+    string s = "info ";
+    foreach (PathSet::const_iterator, i, paths)
+        if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
+    writeLine(run.to, s);
 
-    if (!getIntLine<int>(run.from)) return false;
-    
-    info.deriver = readLine(run.from);
-    if (info.deriver != "") assertStorePath(info.deriver);
-    int nrRefs = getIntLine<int>(run.from);
-    while (nrRefs--) {
-        Path p = readLine(run.from);
-        assertStorePath(p);
-        info.references.insert(p);
+    while (true) {
+        try {
+            Path path = readLine(run.from);
+            if (path == "") break;
+            if (paths.find(path) == paths.end())
+                throw Error(format("got unexpected path `%1%' from substituter") % path);
+            paths.erase(path);
+            SubstitutablePathInfo & info(infos[path]);
+            info.deriver = readLine(run.from);
+            if (info.deriver != "") assertStorePath(info.deriver);
+            int nrRefs = getIntLine<int>(run.from);
+            while (nrRefs--) {
+                Path p = readLine(run.from);
+                assertStorePath(p);
+                info.references.insert(p);
+            }
+            info.downloadSize = getIntLine<long long>(run.from);
+            info.narSize = getIntLine<long long>(run.from);
+        } catch (EndOfFile e) {
+            throw Error(format("substituter `%1%' failed: %2%") % substituter % chomp(drainFD(run.error)));
+        }
     }
-    info.downloadSize = getIntLine<long long>(run.from);
-    info.narSize = getIntLine<long long>(run.from);
-    
-    return true;
 }
 
 
-bool LocalStore::querySubstitutablePathInfo(const Path & path,
-    SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
+    SubstitutablePathInfos & infos)
 {
-    foreach (Paths::iterator, i, substituters)
-        if (querySubstitutablePathInfo(*i, path, info)) return true;
-    return false;
+    PathSet todo = paths;
+    foreach (Paths::iterator, i, settings.substituters) {
+        if (todo.empty()) break;
+        querySubstitutablePathInfos(*i, todo, infos);
+    }
 }
 
 
@@ -1018,11 +1061,10 @@ void LocalStore::registerValidPath(const ValidPathInfo & info)
 
 void LocalStore::registerValidPaths(const ValidPathInfos & infos)
 {
-    /* sqlite will fsync by default, but the new valid paths may not be fsync-ed.
+    /* SQLite will fsync by default, but the new valid paths may not be fsync-ed.
      * So some may want to fsync them before registering the validity, at the
      * expense of some speed of the path registering operation. */
-    if (queryBoolSetting("sync-before-registering", false))
-        sync();
+    if (settings.syncBeforeRegistering) sync();
 
     while (1) {
         try {
@@ -1266,7 +1308,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
         Path hashFile = tmpDir + "/hash";
         writeFile(hashFile, printHash(hash));
 
-        Path secretKey = nixConfDir + "/signing-key.sec";
+        Path secretKey = settings.nixConfDir + "/signing-key.sec";
         checkSecrecy(secretKey);
 
         Strings args;
@@ -1312,7 +1354,7 @@ Path LocalStore::createTempDirInStore()
         /* There is a slight possibility that `tmpDir' gets deleted by
            the GC between createTempDir() and addTempRoot(), so repeat
            until `tmpDir' exists. */
-        tmpDir = createTempDir(nixStore);
+        tmpDir = createTempDir(settings.nixStore);
         addTempRoot(tmpDir);
     } while (!pathExists(tmpDir));
     return tmpDir;
@@ -1364,7 +1406,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
             args.push_back("rsautl");
             args.push_back("-verify");
             args.push_back("-inkey");
-            args.push_back(nixConfDir + "/signing-key.pub");
+            args.push_back(settings.nixConfDir + "/signing-key.pub");
             args.push_back("-pubin");
             args.push_back("-in");
             args.push_back(sigFile);
@@ -1473,13 +1515,13 @@ void LocalStore::verifyStore(bool checkContents)
     /* Acquire the global GC lock to prevent a garbage collection. */
     AutoCloseFD fdGCLock = openGCLock(ltWrite);
 
-    Paths entries = readDirectory(nixStore);
+    Paths entries = readDirectory(settings.nixStore);
     PathSet store(entries.begin(), entries.end());
 
     /* Check whether all valid paths actually exist. */
     printMsg(lvlInfo, "checking path existence...");
 
-    PathSet validPaths2 = queryValidPaths(), validPaths, done;
+    PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
 
     foreach (PathSet::iterator, i, validPaths2)
         verifyPath(*i, store, done, validPaths);
@@ -1583,9 +1625,9 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
 PathSet LocalStore::queryValidPathsOld()
 {
     PathSet paths;
-    Strings entries = readDirectory(nixDBPath + "/info");
+    Strings entries = readDirectory(settings.nixDBPath + "/info");
     foreach (Strings::iterator, i, entries)
-        if (i->at(0) != '.') paths.insert(nixStore + "/" + *i);
+        if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i);
     return paths;
 }
 
@@ -1597,7 +1639,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
 
     /* Read the info file. */
     string baseName = baseNameOf(path);
-    Path infoFile = (format("%1%/info/%2%") % nixDBPath % baseName).str();
+    Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str();
     if (!pathExists(infoFile))
         throw Error(format("path `%1%' is not valid") % path);
     string info = readFile(infoFile);

src/libstore/local-store.hh

@@ -45,7 +45,7 @@ struct OptimiseStats
 struct RunningSubstituter
 {
     Pid pid;
-    AutoCloseFD to, from;
+    AutoCloseFD to, from, error;
 };
 
 
@@ -80,9 +80,6 @@ struct SQLiteStmt
 class LocalStore : public StoreAPI
 {
 private:
-    bool substitutablePathsLoaded;
-    PathSet substitutablePaths;
-
     typedef std::map<Path, RunningSubstituter> RunningSubstituters;
     RunningSubstituters runningSubstituters;
 
@@ -100,7 +97,9 @@ public:
 
     bool isValidPath(const Path & path);
 
-    PathSet queryValidPaths();
+    PathSet queryValidPaths(const PathSet & paths);
+
+    PathSet queryAllValidPaths();
 
     ValidPathInfo queryPathInfo(const Path & path);
 
@@ -124,15 +123,13 @@ public:
 
     Path queryPathFromHashPart(const string & hashPart);
 
-    PathSet querySubstitutablePaths();
+    PathSet querySubstitutablePaths(const PathSet & paths);
 
-    bool hasSubstitutes(const Path & path);
+    void querySubstitutablePathInfos(const Path & substituter,
+        PathSet & paths, SubstitutablePathInfos & infos);
 
-    bool querySubstitutablePathInfo(const Path & path,
-        SubstitutablePathInfo & info);
-    
-    bool querySubstitutablePathInfo(const Path & substituter,
-        const Path & path, SubstitutablePathInfo & info);
+    void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos);
 
     Path addToStore(const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
@@ -293,9 +290,6 @@ void canonicalisePathMetaData(const Path & path, bool recurse);
 
 MakeError(PathInUse, Error);
 
-/* Whether we are in build users mode. */
-bool haveBuildUsers();
-
 /* Whether we are root. */
 bool amPrivileged();
 

src/libstore/misc.cc

@@ -55,45 +55,95 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 
     PathSet todo(targets.begin(), targets.end()), done;
 
-    while (!todo.empty()) {
-        Path p = *(todo.begin());
-        todo.erase(p);
-        if (done.find(p) != done.end()) continue;
-        done.insert(p);
+    /* Getting substitute info has high latency when using the binary
+       cache substituter.  Thus it's essential to do substitute
+       queries in parallel as much as possible.  To accomplish this
+       we do the following:
 
-        if (isDerivation(p)) {
-            if (!store.isValidPath(p)) {
-                unknown.insert(p);
-                continue;
+       - For all paths still to be processed (‘todo’), we add all
+         paths for which we need info to the set ‘query’.  For an
+         unbuilt derivation this is the output paths; otherwise, it's
+         the path itself.
+
+       - We get info about all paths in ‘query’ in parallel.
+
+       - We process the results and add new items to ‘todo’ if
+         necessary.  E.g. if a path is substitutable, then we need to
+         get info on its references.
+
+       - Repeat until ‘todo’ is empty.
+    */
+
+    while (!todo.empty()) {
+
+        PathSet query, todoDrv, todoNonDrv;
+
+        foreach (PathSet::iterator, i, todo) {
+            if (done.find(*i) != done.end()) continue;
+            done.insert(*i);
+
+            if (isDerivation(*i)) {
+                if (!store.isValidPath(*i)) {
+                    // FIXME: we could try to substitute p.
+                    unknown.insert(*i);
+                    continue;
+                }
+                Derivation drv = derivationFromPath(store, *i);
+
+                PathSet invalid;
+                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                    if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
+                if (invalid.empty()) continue;
+
+                todoDrv.insert(*i);
+                if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
             }
-            Derivation drv = derivationFromPath(store, p);
+
+            else {
+                if (store.isValidPath(*i)) continue;
+                query.insert(*i);
+                todoNonDrv.insert(*i);
+            }
+        }
+
+        todo.clear();
+
+        SubstitutablePathInfos infos;
+        store.querySubstitutablePathInfos(query, infos);
+
+        foreach (PathSet::iterator, i, todoDrv) {
+            // FIXME: cache this
+            Derivation drv = derivationFromPath(store, *i);
 
             bool mustBuild = false;
-            foreach (DerivationOutputs::iterator, i, drv.outputs)
-                if (!store.isValidPath(i->second.path) &&
-                    !(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path)))
-                    mustBuild = true;
+            if (settings.useSubstitutes) {
+                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                    if (!store.isValidPath(j->second.path) &&
+                        infos.find(j->second.path) == infos.end())
+                        mustBuild = true;
+            } else
+                mustBuild = true;
 
             if (mustBuild) {
-                willBuild.insert(p);
+                willBuild.insert(*i);
                 todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
                 foreach (DerivationInputs::iterator, i, drv.inputDrvs)
                     todo.insert(i->first);
             } else
                 foreach (DerivationOutputs::iterator, i, drv.outputs)
-                    todo.insert(i->second.path);
+                    todoNonDrv.insert(i->second.path);
         }
 
-        else {
-            if (store.isValidPath(p)) continue;
-            SubstitutablePathInfo info;
-            if (store.querySubstitutablePathInfo(p, info)) {
-                willSubstitute.insert(p);
-                downloadSize += info.downloadSize;
-                narSize += info.narSize;
-                todo.insert(info.references.begin(), info.references.end());
+        foreach (PathSet::iterator, i, todoNonDrv) {
+            done.insert(*i);
+            SubstitutablePathInfos::iterator info = infos.find(*i);
+            if (info != infos.end()) {
+                willSubstitute.insert(*i);
+                downloadSize += info->second.downloadSize;
+                narSize += info->second.narSize;
+                todo.insert(info->second.references.begin(), info->second.references.end());
             } else
-                unknown.insert(p);
+                unknown.insert(*i);
         }
     }
 }

src/libstore/optimise-store.cc

@@ -19,7 +19,7 @@ static void makeWritable(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-	throw SysError(format("getting attributes of path `%1%'") % path);
+        throw SysError(format("getting attributes of path `%1%'") % path);
     if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
         throw SysError(format("changing writability of `%1%'") % path);
@@ -57,12 +57,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
     
     struct stat st;
     if (lstat(path.c_str(), &st))
-	throw SysError(format("getting attributes of path `%1%'") % path);
+        throw SysError(format("getting attributes of path `%1%'") % path);
 
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
-	foreach (Strings::iterator, i, names)
-	    optimisePath_(stats, path + "/" + *i);
+        foreach (Strings::iterator, i, names)
+            optimisePath_(stats, path + "/" + *i);
         return;
     }
 
@@ -113,7 +113,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
        current file with a hard link to that file. */
     struct stat stLink;
     if (lstat(linkPath.c_str(), &stLink))
-	throw SysError(format("getting attributes of path `%1%'") % linkPath);
+        throw SysError(format("getting attributes of path `%1%'") % linkPath);
 
     stats.sameContents++;
     if (st.st_ino == stLink.st_ino) {
@@ -149,7 +149,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
         MakeImmutable mk1(linkPath);
 
         Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
-            % nixStore % getpid() % rand()).str();
+            % settings.nixStore % getpid() % rand()).str();
 
         if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
             if (errno == EMLINK) {
@@ -194,7 +194,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
 
 void LocalStore::optimiseStore(OptimiseStats & stats)
 {
-    PathSet paths = queryValidPaths();
+    PathSet paths = queryAllValidPaths();
 
     foreach (PathSet::iterator, i, paths) {
         addTempRoot(*i);
@@ -207,10 +207,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
 
 void LocalStore::optimisePath(const Path & path)
 {
-    if (queryBoolSetting("auto-optimise-store", true)) {
-        OptimiseStats stats;
-        optimisePath_(stats, path);
-    }
+    OptimiseStats stats;
+    if (settings.autoOptimiseStore) optimisePath_(stats, path);
 }
 
 

src/libstore/remote-store.cc

@@ -100,7 +100,7 @@ void RemoteStore::forkSlave()
     /* Start the worker. */
     Path worker = getEnv("NIX_WORKER");
     if (worker == "")
-        worker = nixBinDir + "/nix-worker";
+        worker = settings.nixBinDir + "/nix-worker";
 
     child = fork();
 
@@ -142,7 +142,7 @@ void RemoteStore::connectToDaemon()
     if (fdSocket == -1)
         throw SysError("cannot create Unix domain socket");
 
-    string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
+    string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
 
     /* Urgh, sockaddr_un allows path names of only 108 characters.  So
        chdir to the socket directory so that we can pass a relative
@@ -184,23 +184,33 @@ RemoteStore::~RemoteStore()
 void RemoteStore::setOptions()
 {
     writeInt(wopSetOptions, to);
-    writeInt(keepFailed, to);
-    writeInt(keepGoing, to);
-    writeInt(tryFallback, to);
+
+    writeInt(settings.keepFailed, to);
+    writeInt(settings.keepGoing, to);
+    writeInt(settings.tryFallback, to);
     writeInt(verbosity, to);
-    writeInt(maxBuildJobs, to);
-    writeInt(maxSilentTime, to);
+    writeInt(settings.maxBuildJobs, to);
+    writeInt(settings.maxSilentTime, to);
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
-        writeInt(useBuildHook, to);
+        writeInt(settings.useBuildHook, to);
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) {
-        writeInt(buildVerbosity, to);
+        writeInt(settings.buildVerbosity, to);
         writeInt(logType, to);
-        writeInt(printBuildTrace, to);
+        writeInt(settings.printBuildTrace, to);
     }
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 6)
-        writeInt(buildCores, to);
+        writeInt(settings.buildCores, to);
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 10)
-        writeInt(queryBoolSetting("build-use-substitutes", true), to);
+        writeInt(settings.useSubstitutes, to);
+
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) {
+        Settings::SettingsMap overrides = settings.getOverrides();
+        writeInt(overrides.size(), to);
+        foreach (Settings::SettingsMap::iterator, i, overrides) {
+            writeString(i->first, to);
+            writeString(i->second, to);
+        }
+    }
 
     processStderr();
 }
@@ -217,42 +227,96 @@ bool RemoteStore::isValidPath(const Path & path)
 }
 
 
-PathSet RemoteStore::queryValidPaths()
+PathSet RemoteStore::queryValidPaths(const PathSet & paths)
 {
     openConnection();
-    writeInt(wopQueryValidPaths, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+        PathSet res;
+        foreach (PathSet::const_iterator, i, paths)
+            if (isValidPath(*i)) res.insert(*i);
+        return res;
+    } else {
+        writeInt(wopQueryValidPaths, to);
+        writeStrings(paths, to);
+        processStderr();
+        return readStorePaths<PathSet>(from);
+    }
+}
+
+
+PathSet RemoteStore::queryAllValidPaths()
+{
+    openConnection();
+    writeInt(wopQueryAllValidPaths, to);
     processStderr();
     return readStorePaths<PathSet>(from);
 }
 
 
-bool RemoteStore::hasSubstitutes(const Path & path)
+PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
 {
     openConnection();
-    writeInt(wopHasSubstitutes, to);
-    writeString(path, to);
-    processStderr();
-    unsigned int reply = readInt(from);
-    return reply != 0;
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+        PathSet res;
+        foreach (PathSet::const_iterator, i, paths) {
+            writeInt(wopHasSubstitutes, to);
+            writeString(*i, to);
+            processStderr();
+            if (readInt(from)) res.insert(*i);
+        }
+        return res;
+    } else {
+        writeInt(wopQuerySubstitutablePaths, to);
+        writeStrings(paths, to);
+        processStderr();
+        return readStorePaths<PathSet>(from);
+    }
 }
 
 
-bool RemoteStore::querySubstitutablePathInfo(const Path & path,
-    SubstitutablePathInfo & info)
+void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
+    SubstitutablePathInfos & infos)
 {
+    if (paths.empty()) return;
+
     openConnection();
-    if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false;
-    writeInt(wopQuerySubstitutablePathInfo, to);
-    writeString(path, to);
-    processStderr();
-    unsigned int reply = readInt(from);
-    if (reply == 0) return false;
-    info.deriver = readString(from);
-    if (info.deriver != "") assertStorePath(info.deriver);
-    info.references = readStorePaths<PathSet>(from);
-    info.downloadSize = readLongLong(from);
-    info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
-    return true;
+
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return;
+
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+
+        foreach (PathSet::const_iterator, i, paths) {
+            SubstitutablePathInfo info;
+            writeInt(wopQuerySubstitutablePathInfo, to);
+            writeString(*i, to);
+            processStderr();
+            unsigned int reply = readInt(from);
+            if (reply == 0) continue;
+            info.deriver = readString(from);
+            if (info.deriver != "") assertStorePath(info.deriver);
+            info.references = readStorePaths<PathSet>(from);
+            info.downloadSize = readLongLong(from);
+            info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
+            infos[*i] = info;
+        }
+
+    } else {
+
+        writeInt(wopQuerySubstitutablePathInfos, to);
+        writeStrings(paths, to);
+        processStderr();
+        unsigned int count = readInt(from);
+        for (unsigned int n = 0; n < count; n++) {
+            Path path = readStorePath(from);
+            SubstitutablePathInfo & info(infos[path]);
+            info.deriver = readString(from);
+            if (info.deriver != "") assertStorePath(info.deriver);
+            info.references = readStorePaths<PathSet>(from);
+            info.downloadSize = readLongLong(from);
+            info.narSize = readLongLong(from);
+        }
+
+    }
 }
 
 

src/libstore/remote-store.hh

@@ -26,7 +26,9 @@ public:
     
     bool isValidPath(const Path & path);
 
-    PathSet queryValidPaths();
+    PathSet queryValidPaths(const PathSet & paths);
+    
+    PathSet queryAllValidPaths();
     
     ValidPathInfo queryPathInfo(const Path & path);
 
@@ -44,10 +46,10 @@ public:
 
     Path queryPathFromHashPart(const string & hashPart);
     
-    bool hasSubstitutes(const Path & path);
+    PathSet querySubstitutablePaths(const PathSet & paths);
     
-    bool querySubstitutablePathInfo(const Path & path,
-        SubstitutablePathInfo & info);
+    void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos);
     
     Path addToStore(const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,

src/libstore/store-api.cc

@@ -19,16 +19,16 @@ GCOptions::GCOptions()
 bool isInStore(const Path & path)
 {
     return path[0] == '/'
-        && string(path, 0, nixStore.size()) == nixStore
-        && path.size() >= nixStore.size() + 2
-        && path[nixStore.size()] == '/';
+        && string(path, 0, settings.nixStore.size()) == settings.nixStore
+        && path.size() >= settings.nixStore.size() + 2
+        && path[settings.nixStore.size()] == '/';
 }
 
 
 bool isStorePath(const Path & path)
 {
     return isInStore(path)
-        && path.find('/', nixStore.size() + 1) == Path::npos;
+        && path.find('/', settings.nixStore.size() + 1) == Path::npos;
 }
 
 
@@ -43,7 +43,7 @@ Path toStorePath(const Path & path)
 {
     if (!isInStore(path))
         throw Error(format("path `%1%' is not in the Nix store") % path);
-    Path::size_type slash = path.find('/', nixStore.size() + 1);
+    Path::size_type slash = path.find('/', settings.nixStore.size() + 1);
     if (slash == Path::npos)
         return path;
     else
@@ -74,7 +74,7 @@ Path followLinksToStorePath(const Path & path)
 string storePathToName(const Path & path)
 {
     assertStorePath(path);
-    return string(path, nixStore.size() + 34);
+    return string(path, settings.nixStore.size() + 34);
 }
 
 
@@ -173,11 +173,11 @@ Path makeStorePath(const string & type,
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
     string s = type + ":sha256:" + printHash(hash) + ":"
-        + nixStore + ":" + name;
+        + settings.nixStore + ":" + name;
 
     checkStoreName(name);
 
-    return nixStore + "/"
+    return settings.nixStore + "/"
         + printHash32(compressHash(hashString(htSHA256, s), 20))
         + "-" + name;
 }

src/libstore/store-api.hh

@@ -80,6 +80,8 @@ struct SubstitutablePathInfo
     unsigned long long narSize; /* 0 = unknown */
 };
 
+typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
+
 
 struct ValidPathInfo 
 {
@@ -102,20 +104,23 @@ public:
 
     virtual ~StoreAPI() { }
 
-    /* Checks whether a path is valid. */ 
+    /* Check whether a path is valid. */ 
     virtual bool isValidPath(const Path & path) = 0;
 
-    /* Query the set of valid paths. */
-    virtual PathSet queryValidPaths() = 0;
+    /* Query which of the given paths is valid. */
+    virtual PathSet queryValidPaths(const PathSet & paths) = 0;
+
+    /* Query the set of all valid paths. */
+    virtual PathSet queryAllValidPaths() = 0;
 
     /* Query information about a valid path. */
     virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
 
-    /* Queries the hash of a valid path. */ 
+    /* Query the hash of a valid path. */ 
     virtual Hash queryPathHash(const Path & path) = 0;
 
-    /* Queries the set of outgoing FS references for a store path.
-       The result is not cleared. */
+    /* Query the set of outgoing FS references for a store path.  The
+       result is not cleared. */
     virtual void queryReferences(const Path & path,
         PathSet & references) = 0;
 
@@ -138,13 +143,14 @@ public:
        path, or "" if the path doesn't exist. */
     virtual Path queryPathFromHashPart(const string & hashPart) = 0;
     
-    /* Query whether a path has substitutes. */
-    virtual bool hasSubstitutes(const Path & path) = 0;
+    /* Query which of the given paths have substitutes. */
+    virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
 
-    /* Query the references, deriver and download size of a
-       substitutable path. */
-    virtual bool querySubstitutablePathInfo(const Path & path,
-        SubstitutablePathInfo & info) = 0;
+    /* Query substitute info (i.e. references, derivers and download
+       sizes) of a set of paths.  If a path does not have substitute
+       info, it's omitted from the resulting ‘infos’ map. */
+    virtual void querySubstitutablePathInfos(const PathSet & paths,
+        SubstitutablePathInfos & infos) = 0;
     
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.

src/libstore/worker-protocol.hh

@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x10b
+#define PROTOCOL_VERSION 0x10c
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -32,13 +32,16 @@ typedef enum {
     wopCollectGarbage = 20,
     wopQuerySubstitutablePathInfo = 21,
     wopQueryDerivationOutputs = 22,
-    wopQueryValidPaths = 23,
+    wopQueryAllValidPaths = 23,
     wopQueryFailedPaths = 24,
     wopClearFailedPaths = 25,
     wopQueryPathInfo = 26,
     wopImportPaths = 27,
     wopQueryDerivationOutputNames = 28,
     wopQueryPathFromHashPart = 29,
+    wopQuerySubstitutablePathInfos = 30,
+    wopQueryValidPaths = 31,
+    wopQuerySubstitutablePaths = 32,
 } WorkerOp;
 
 

src/libutil/util.cc

@@ -253,7 +253,7 @@ string readLine(int fd)
             if (errno != EINTR)
                 throw SysError("reading a line");
         } else if (rd == 0)
-            throw Error("unexpected EOF reading a line");
+            throw EndOfFile("unexpected EOF reading a line");
         else {
             if (ch == '\n') return s;
             s += ch;
@@ -1010,6 +1010,13 @@ string concatStringsSep(const string & sep, const Strings & ss)
 }
 
 
+string chomp(const string & s)
+{
+    size_t i = s.find_last_not_of(" \n\r\t");
+    return i == string::npos ? "" : string(s, 0, i + 1);
+}
+
+
 string statusToString(int status)
 {
     if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

src/libutil/util.hh

@@ -294,6 +294,10 @@ Strings tokenizeString(const string & s, const string & separators = " \t\n\r");
 string concatStringsSep(const string & sep, const Strings & ss);
 
 
+/* Remove trailing whitespace from a string. */
+string chomp(const string & s);
+
+
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);

src/nix-env/nix-env.cc

@@ -55,7 +55,6 @@ struct Globals
     EvalState state;
     bool dryRun;
     bool preserveInstalled;
-    bool keepDerivations;
     string forceName;
     bool prebuiltOnly;
 };
@@ -113,6 +112,11 @@ static void getAllExprs(EvalState & state,
     StringSet namesSorted(names.begin(), names.end());
 
     foreach (StringSet::iterator, i, namesSorted) {
+        /* Ignore the manifest.nix used by profiles.  This is
+           necessary to prevent it from showing up in channels (which
+           are implemented using profiles). */
+        if (*i == "manifest.nix") continue;
+        
         Path path2 = path + "/" + *i;
         
         struct stat st;
@@ -211,9 +215,12 @@ static int comparePriorities(EvalState & state,
 
 static bool isPrebuilt(EvalState & state, const DrvInfo & elem)
 {
+    assert(false);
+#if 0
     return
         store->isValidPath(elem.queryOutPath(state)) ||
         store->hasSubstitutes(elem.queryOutPath(state));
+#endif
 }
 
 
@@ -263,8 +270,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
                 
                 if (k != newest.end()) {
                     d = j->first.system == k->second.first.system ? 0 :
-                        j->first.system == thisSystem ? 1 :
-                        k->second.first.system == thisSystem ? -1 : 0;
+                        j->first.system == settings.thisSystem ? 1 :
+                        k->second.first.system == settings.thisSystem ? -1 : 0;
                     if (d == 0)
                         d = comparePriorities(state, j->first, k->second.first);
                     if (d == 0)
@@ -495,7 +502,7 @@ static void installDerivations(Globals & globals,
         if (globals.dryRun) return;
 
         if (createUserEnv(globals.state, allElems,
-                profile, globals.keepDerivations, lockToken)) break;
+                profile, settings.envKeepDerivations, lockToken)) break;
     }
 }
 
@@ -602,7 +609,7 @@ static void upgradeDerivations(Globals & globals,
         if (globals.dryRun) return;
 
         if (createUserEnv(globals.state, newElems,
-                globals.profile, globals.keepDerivations, lockToken)) break;
+                globals.profile, settings.envKeepDerivations, lockToken)) break;
     }
 }
 
@@ -669,7 +676,7 @@ static void opSetFlag(Globals & globals,
 
         /* Write the new user environment. */
         if (createUserEnv(globals.state, installedElems,
-                globals.profile, globals.keepDerivations, lockToken)) break;
+                globals.profile, settings.envKeepDerivations, lockToken)) break;
     }
 }
 
@@ -737,7 +744,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
         if (globals.dryRun) return;
 
         if (createUserEnv(globals.state, newElems,
-                profile, globals.keepDerivations, lockToken)) break;
+                profile, settings.envKeepDerivations, lockToken)) break;
     }
 }
 
@@ -866,7 +873,7 @@ static void opQuery(Globals & globals,
 
     enum { sInstalled, sAvailable } source = sInstalled;
 
-    readOnlyMode = true; /* makes evaluation a bit faster */
+    settings.readOnlyMode = true; /* makes evaluation a bit faster */
 
     for (Strings::iterator i = args.begin(); i != args.end(); ) {
         string arg = *i++;
@@ -930,6 +937,22 @@ static void opQuery(Globals & globals,
     }
 
 
+    /* Query which paths have substitutes. */
+    PathSet validPaths, substitutablePaths;
+    if (printStatus) {
+        PathSet paths;
+        foreach (vector<DrvInfo>::iterator, i, elems2)
+            try {
+                paths.insert(i->queryOutPath(globals.state));
+            } catch (AssertionError & e) {
+                printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name);
+                i->setFailed();
+            }
+        validPaths = store->queryValidPaths(paths);
+        substitutablePaths = store->querySubstitutablePaths(paths);
+    }
+
+    
     /* Print the desired columns, or XML output. */
     Table table;
     std::ostringstream dummy;
@@ -938,6 +961,8 @@ static void opQuery(Globals & globals,
     
     foreach (vector<DrvInfo>::iterator, i, elems2) {
         try {
+            if (i->hasFailed()) continue;
+            
             startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath);
 
             if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue;
@@ -949,9 +974,10 @@ static void opQuery(Globals & globals,
             XMLAttrs attrs;
 
             if (printStatus) {
-                bool hasSubs = store->hasSubstitutes(i->queryOutPath(globals.state));
-                bool isInstalled = installed.find(i->queryOutPath(globals.state)) != installed.end();
-                bool isValid = store->isValidPath(i->queryOutPath(globals.state));
+                Path outPath = i->queryOutPath(globals.state);
+                bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end();
+                bool isInstalled = installed.find(outPath) != installed.end();
+                bool isValid = validPaths.find(outPath) != validPaths.end();
                 if (xmlOutput) {
                     attrs["installed"] = isInstalled ? "1" : "0";
                     attrs["valid"] = isValid ? "1" : "0";
@@ -1240,9 +1266,6 @@ void run(Strings args)
     globals.preserveInstalled = false;
     globals.prebuiltOnly = false;
 
-    globals.keepDerivations =
-        queryBoolSetting("env-keep-derivations", false);
-    
     for (Strings::iterator i = args.begin(); i != args.end(); ) {
         string arg = *i++;
 
@@ -1309,7 +1332,7 @@ void run(Strings args)
         Path profileLink = getHomeDir() + "/.nix-profile";
         globals.profile = pathExists(profileLink)
             ? absPath(readLink(profileLink), dirOf(profileLink))
-            : canonPath(nixStateDir + "/profiles/default");
+            : canonPath(settings.nixStateDir + "/profiles/default");
     }
     
     store = openStore();

src/nix-instantiate/nix-instantiate.cc

@@ -96,11 +96,11 @@ void run(Strings args)
         if (arg == "-")
             readStdin = true;
         else if (arg == "--eval-only") {
-            readOnlyMode = true;
+            settings.readOnlyMode = true;
             evalOnly = true;
         }
         else if (arg == "--parse-only") {
-            readOnlyMode = true;
+            settings.readOnlyMode = true;
             parseOnly = evalOnly = true;
         }
         else if (arg == "--find-file")

src/nix-store/nix-store.cc

@@ -432,7 +432,7 @@ static void opReadLog(Strings opFlags, Strings opArgs)
         Path path = useDeriver(followLinksToStorePath(*i));
 
         Path logPath = (format("%1%/%2%/%3%") %
-            nixLogDir % drvsLogDir % baseNameOf(path)).str();
+            settings.nixLogDir % drvsLogDir % baseNameOf(path)).str();
         Path logBz2Path = logPath + ".bz2";
 
         if (pathExists(logPath)) {
@@ -469,7 +469,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
-    PathSet validPaths = store->queryValidPaths();
+    PathSet validPaths = store->queryAllValidPaths();
     foreach (PathSet::iterator, i, validPaths)
         cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
 }

src/nix-worker/nix-worker.cc

@@ -297,12 +297,30 @@ static void performOp(unsigned int clientVersion,
         break;
     }
 
+    case wopQueryValidPaths: {
+        PathSet paths = readStorePaths<PathSet>(from);
+        startWork();
+        PathSet res = store->queryValidPaths(paths);
+        stopWork();
+        writeStrings(res, to);
+        break;
+    }
+
     case wopHasSubstitutes: {
         Path path = readStorePath(from);
         startWork();
-        bool result = store->hasSubstitutes(path);
+        PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path));
         stopWork();
-        writeInt(result, to);
+        writeInt(res.find(path) != res.end(), to);
+        break;
+    }
+
+    case wopQuerySubstitutablePaths: {
+        PathSet paths = readStorePaths<PathSet>(from);
+        startWork();
+        PathSet res = store->querySubstitutablePaths(paths);
+        stopWork();
+        writeStrings(res, to);
         break;
     }
 
@@ -509,26 +527,30 @@ static void performOp(unsigned int clientVersion,
     }
 
     case wopSetOptions: {
-        keepFailed = readInt(from) != 0;
-        keepGoing = readInt(from) != 0;
-        tryFallback = readInt(from) != 0;
+        settings.keepFailed = readInt(from) != 0;
+        settings.keepGoing = readInt(from) != 0;
+        settings.tryFallback = readInt(from) != 0;
         verbosity = (Verbosity) readInt(from);
-        maxBuildJobs = readInt(from);
-        maxSilentTime = readInt(from);
+        settings.maxBuildJobs = readInt(from);
+        settings.maxSilentTime = readInt(from);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
-            useBuildHook = readInt(from) != 0;
+            settings.useBuildHook = readInt(from) != 0;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 4) {
-            buildVerbosity = (Verbosity) readInt(from);
+            settings.buildVerbosity = (Verbosity) readInt(from);
             logType = (LogType) readInt(from);
-            printBuildTrace = readInt(from) != 0;
+            settings.printBuildTrace = readInt(from) != 0;
         }
         if (GET_PROTOCOL_MINOR(clientVersion) >= 6)
-            buildCores = readInt(from);
-        if (GET_PROTOCOL_MINOR(clientVersion) >= 10) {
-            int x = readInt(from);
-            Strings ss;
-            ss.push_back(x == 0 ? "false" : "true");
-            overrideSetting("build-use-substitutes", ss);
+            settings.buildCores = readInt(from);
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 10)
+            settings.useSubstitutes = readInt(from) != 0;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
+            unsigned int n = readInt(from);
+            for (unsigned int i = 0; i < n; i++) {
+                string name = readString(from);
+                string value = readString(from);
+                settings.set("untrusted-" + name, value);
+            }
         }
         startWork();
         stopWork();
@@ -538,23 +560,43 @@ static void performOp(unsigned int clientVersion,
     case wopQuerySubstitutablePathInfo: {
         Path path = absPath(readString(from));
         startWork();
-        SubstitutablePathInfo info;
-        bool res = store->querySubstitutablePathInfo(path, info);
+        SubstitutablePathInfos infos;
+        store->querySubstitutablePathInfos(singleton<PathSet>(path), infos);
         stopWork();
-        writeInt(res ? 1 : 0, to);
-        if (res) {
-            writeString(info.deriver, to);
-            writeStrings(info.references, to);
-            writeLongLong(info.downloadSize, to);
+        SubstitutablePathInfos::iterator i = infos.find(path);
+        if (i == infos.end())
+            writeInt(0, to);
+        else {
+            writeInt(1, to);
+            writeString(i->second.deriver, to);
+            writeStrings(i->second.references, to);
+            writeLongLong(i->second.downloadSize, to);
             if (GET_PROTOCOL_MINOR(clientVersion) >= 7)
-                writeLongLong(info.narSize, to);
+                writeLongLong(i->second.narSize, to);
         }
         break;
     }
 
-    case wopQueryValidPaths: {
+    case wopQuerySubstitutablePathInfos: {
+        PathSet paths = readStorePaths<PathSet>(from);
         startWork();
-        PathSet paths = store->queryValidPaths();
+        SubstitutablePathInfos infos;
+        store->querySubstitutablePathInfos(paths, infos);
+        stopWork();
+        writeInt(infos.size(), to);
+        foreach (SubstitutablePathInfos::iterator, i, infos) {
+            writeString(i->first, to);
+            writeString(i->second.deriver, to);
+            writeStrings(i->second.references, to);
+            writeLongLong(i->second.downloadSize, to);
+            writeLongLong(i->second.narSize, to);
+        }
+        break;
+    }
+
+    case wopQueryAllValidPaths: {
+        startWork();
+        PathSet paths = store->queryAllValidPaths();
         stopWork();
         writeStrings(paths, to);
         break;
@@ -730,7 +772,7 @@ static void daemonLoop()
         if (fdSocket == -1)
             throw SysError("cannot create Unix domain socket");
 
-        string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
+        string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
 
         createDirs(dirOf(socketPath));
 
@@ -781,10 +823,10 @@ static void daemonLoop()
                 (struct sockaddr *) &remoteAddr, &remoteAddrLen);
             checkInterrupt();
             if (remote == -1) {
-		if (errno == EINTR)
-		    continue;
-		else
-		    throw SysError("accepting connection");
+                if (errno == EINTR)
+                    continue;
+                else
+                    throw SysError("accepting connection");
             }
 
             closeOnExec(remote);
@@ -829,10 +871,6 @@ static void daemonLoop()
                         strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
                     }
 
-                    /* Since the daemon can be long-running, the
-                       settings may have changed.  So force a reload. */
-                    reloadSettings();
-                    
                     /* Handle the connection. */
                     from.fd = remote;
                     to.fd = remote;

substitute.mk

@@ -16,11 +16,13 @@
 	 -e "s^@shell\@^$(bash)^g" \
 	 -e "s^@curl\@^$(curl)^g" \
 	 -e "s^@bzip2\@^$(bzip2)^g" \
+	 -e "s^@xz\@^$(xz)^g" \
 	 -e "s^@perl\@^$(perl)^g" \
 	 -e "s^@perlFlags\@^$(perlFlags)^g" \
 	 -e "s^@coreutils\@^$(coreutils)^g" \
 	 -e "s^@sed\@^$(sed)^g" \
 	 -e "s^@tar\@^$(tar)^g" \
+	 -e "s^@tarFlags\@^$(tarFlags)^g" \
 	 -e "s^@gzip\@^$(gzip)^g" \
 	 -e "s^@pv\@^$(pv)^g" \
 	 -e "s^@tr\@^$(tr)^g" \

tests/Makefile.am

@@ -9,7 +9,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
   gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
   remote-store.sh export.sh export-graph.sh negative-caching.sh \
   binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
-  multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh
+  multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
+  binary-cache.sh
 
 XFAIL_TESTS =
 

tests/binary-cache.sh

@@ -0,0 +1,35 @@
+source common.sh
+
+clearStore
+
+# Create the binary cache.
+cacheDir=$TEST_ROOT/binary-cache
+rm -rf $cacheDir
+
+outPath=$(nix-build dependencies.nix --no-out-link)
+
+nix-push --dest $cacheDir $outPath
+
+
+# By default, a binary cache doesn't support "nix-env -qas", but does
+# support installation.
+clearStore
+rm -f $NIX_STATE_DIR/binary-cache*
+
+nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
+
+nix-store --option binary-caches "file://$cacheDir" -r $outPath
+
+
+# But with the right configuration, "nix-env -qas" should also work.
+clearStore
+rm -f $NIX_STATE_DIR/binary-cache*
+echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
+
+nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
+
+nix-store --option binary-caches "file://$cacheDir" -r $outPath
+
+nix-store --check-validity $outPath
+nix-store -qR $outPath | grep input-2
+

tests/binary-patching.sh

@@ -7,14 +7,17 @@ mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches
 RESULT=$TEST_ROOT/result
 
 # Build version 1 and 2 of the "foo" package.
-nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest1 \
+nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 \
     $(nix-build -o $RESULT binary-patching.nix --arg version 1)
+mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest1 
 
 out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2)
-nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest2 $out2
+nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out2
+mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest2
     
 out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3)
-nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest3 $out3
+nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out3
+mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest3
 
 rm $RESULT
 

tests/common.sh.in

@@ -16,6 +16,7 @@ export NIX_DB_DIR=$TEST_ROOT/db
 export NIX_CONF_DIR=$TEST_ROOT/etc
 export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
 export SHARED=$TEST_ROOT/shared
+export NIX_REMOTE=$NIX_REMOTE_
 
 export PATH=@bindir@:$PATH
 
@@ -79,3 +80,5 @@ fail() {
     echo "$1"
     exit 1
 }
+
+set -x

tests/install-package.sh

@@ -9,7 +9,7 @@ clearStore
 clearProfiles
 
 cat > $TEST_ROOT/foo.nixpkg <<EOF
-NIXPKG1 file://$TEST_ROOT/manifest simple $system $drvPath $outPath
+NIXPKG1 file://$TEST_ROOT/cache/MANIFEST simple $system $drvPath $outPath
 EOF
 
 nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg

tests/nix-channel.sh

@@ -19,7 +19,7 @@ nix-channel --remove xyzzy
 # Create a channel.
 rm -rf $TEST_ROOT/foo
 mkdir -p $TEST_ROOT/foo
-nix-push --copy $TEST_ROOT/foo $TEST_ROOT/foo/MANIFEST $(nix-store -r $(nix-instantiate dependencies.nix))
+nix-push --dest $TEST_ROOT/foo --manifest --bzip2 $(nix-store -r $(nix-instantiate dependencies.nix))
 rm -rf $TEST_ROOT/nixexprs
 mkdir -p $TEST_ROOT/nixexprs
 cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/

tests/nix-pull.sh

@@ -2,7 +2,7 @@ source common.sh
 
 pullCache () {
     echo "pulling cache..."
-    nix-pull file://$TEST_ROOT/manifest
+    nix-pull file://$TEST_ROOT/cache/MANIFEST
 }
 
 clearStore

tests/nix-push.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+clearStore
+
 drvPath=$(nix-instantiate dependencies.nix)
 outPath=$(nix-store -r $drvPath)
 
@@ -7,4 +9,4 @@ echo "pushing $drvPath"
 
 mkdir -p $TEST_ROOT/cache
 
-nix-push --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath
+nix-push --dest $TEST_ROOT/cache --manifest $drvPath --bzip2

tests/referrers.sh

@@ -10,6 +10,7 @@ touch $reference
 
 echo "making registration..."
 
+set +x
 for ((n = 0; n < $max; n++)); do
     storePath=$NIX_STORE_DIR/$n
     echo -n > $storePath
@@ -19,6 +20,7 @@ for ((n = 0; n < $max; n++)); do
     fi
     echo $storePath; echo; echo 2; echo $reference; echo $ref2
 done > $TEST_ROOT/reg_info
+set -x
 
 echo "registering..."
 

tests/remote-store.sh

@@ -3,7 +3,7 @@ source common.sh
 echo '*** testing slave mode ***'
 clearStore
 clearManifests
-NIX_REMOTE=slave $SHELL ./user-envs.sh
+NIX_REMOTE_=slave $SHELL ./user-envs.sh
 
 echo '*** testing daemon mode ***'
 clearStore

tests/substituter.sh

@@ -2,22 +2,25 @@
 echo substituter args: $* >&2
 
 if test $1 = "--query"; then
-    while read cmd; do
-        echo FOO $cmd >&2
+    while read cmd args; do
+        echo "CMD = $cmd, ARGS = $args" >&2
         if test "$cmd" = "have"; then
-            read path
-            if grep -q "$path" $TEST_ROOT/sub-paths; then
-                echo 1
-            else
-                echo 0
-            fi
+            for path in $args; do 
+                read path
+                if grep -q "$path" $TEST_ROOT/sub-paths; then
+                    echo $path
+                fi
+            done
+            echo
         elif test "$cmd" = "info"; then
-            read path
-            echo 1
-            echo "" # deriver
-            echo 0 # nr of refs
-            echo $((1 * 1024 * 1024)) # download size
-            echo $((2 * 1024 * 1024)) # nar size
+            for path in $args; do
+                echo $path
+                echo "" # deriver
+                echo 0 # nr of refs
+                echo $((1 * 1024 * 1024)) # download size
+                echo $((2 * 1024 * 1024)) # nar size
+            done
+            echo
         else
             echo "bad command $cmd"
             exit 1
@@ -26,6 +29,7 @@ if test $1 = "--query"; then
 elif test $1 = "--substitute"; then
     mkdir $2
     echo "Hallo Wereld" > $2/hello
+    echo # no expected hash
 else
     echo "unknown substituter operation"
     exit 1

tests/substituter2.sh

@@ -2,21 +2,23 @@
 echo substituter2 args: $* >&2
 
 if test $1 = "--query"; then
-    while read cmd; do
-        if test "$cmd" = "have"; then
-            read path
-            if grep -q "$path" $TEST_ROOT/sub-paths; then
-                echo 1
-            else
-                echo 0
-            fi
-        elif test "$cmd" = "info"; then
-            read path
-            echo 1
-            echo "" # deriver
-            echo 0 # nr of refs
-            echo 0 # download size
-            echo 0 # nar size
+    while read cmd args; do
+        if test "$cmd" = have; then
+            for path in $args; do
+                if grep -q "$path" $TEST_ROOT/sub-paths; then
+                    echo $path
+                fi
+            done
+            echo
+        elif test "$cmd" = info; then
+            for path in $args; do
+                echo $path
+                echo "" # deriver
+                echo 0 # nr of refs
+                echo 0 # download size
+                echo 0 # nar size
+            done
+            echo
         else
             echo "bad command $cmd"
             exit 1