* Prebuilt package sharing. We allow transparent binary deployment by

sharing package directories (i.e., the result of building a Nix
  descriptor).

  `nix-pull-prebuilts' obtains a list of all known prebuilts by
  consulting the paths and URLs specified in
  $prefix/etc/nix/prebuilts.conf.  The mappings ($pkghash,
  $prebuilthash) and ($prebuilthash, $location) are registered with
  Nix so that it can use the prebuilt with hash $prebuilthash when
  installing a package with hash $pkghash by downloading and unpacking
  $location.

  `nix-push-prebuilts' creates prebuilts for all packages for which no
  prebuilt is known to exist.  It can then optionally upload these
  to the network through rsync.

  `nix-[pull|push]-prebuilts' just provide a policy.  Nix provides the
  mechanism through the `nix [export|regprebuilt|regurl]' commands.

scripts/Makefile.am

@@ -1,5 +1,9 @@
-bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage
+bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage \
+ nix-pull-prebuilts nix-push-prebuilts
 
 install-exec-local:
 	$(INSTALL) -d $(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh
+	$(INSTALL) -d $(sysconfdir)/nix
+    # !!! don't overwrite local modifications
+	$(INSTALL_PROGRAM) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf

scripts/nix-generate-regscript

@@ -1,20 +0,0 @@
-#! /usr/bin/perl -w
-
-my $dir = shift @ARGV;
-$dir || die "missing directory";
-my $url = shift @ARGV;
-$url || die "missing base url";
-
-chdir $dir || die "cannot chdir to $dir";
-
-foreach my $prebuilt (glob("*.tar.bz2")) {
-
-    $prebuilt =~ /-([a-z0-9]+)-([a-z0-9]+).tar.bz2$/
-	|| die "invalid file name: $prebuilt";
-
-    my $pkgHash = $1;
-    my $prebuiltHash = $2;
-
-    print "regprebuilt $pkgHash $prebuiltHash\n";
-    print "regurl $prebuiltHash $url/$prebuilt\n";
-}

scripts/nix-pull-prebuilts

@@ -0,0 +1,69 @@
+#! /usr/bin/perl -w
+
+my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
+my $etcdir = "$prefix/etc/nix";
+my $knowns = "$prefix/var/nix/known-prebuilts";
+my $tmpfile = "$prefix/var/nix/prebuilts.tmp";
+
+my $conffile = "$etcdir/prebuilts.conf";
+
+sub register {
+    my $fn = shift;
+    return unless $fn =~ /([^\/]*)-([0-9a-z]{32})-([0-9a-z]{32})\.tar\.bz2/;
+    my $id = $1;
+    my $pkghash = $2;
+    my $prebuilthash = $3;
+    print "$pkghash => $prebuilthash ($id)\n";
+    system "nix regprebuilt $pkghash $prebuilthash";
+    if ($?) { die "`nix regprebuilt' failed"; }
+    print KNOWNS "$pkghash\n";
+}
+
+open KNOWNS, ">$knowns";
+
+open CONFFILE, "<$conffile";
+
+while (<CONFFILE>) {
+    chomp;
+    if (/^\s*(\S+)\s*(\#.*)?$/) {
+        my $url = $1;
+
+        print "obtaining prebuilt list from $url...\n";
+
+        if ($url =~ /^\//) {
+
+            # It's a local path.
+
+            foreach my $fn (glob "$url/*") {
+                register $fn;
+            }
+
+        } else {
+
+            # It's a URL.
+
+            system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape
+            if ($?) { die "`wget' failed"; }
+
+            open INDEX, "<$tmpfile";
+
+            while (<INDEX>) {
+                # Get all links to prebuilts, that is, file names of the
+                # form foo-HASH-HASH.tar.bz2.
+                next unless (/HREF=\"([^\"]*)\"/);
+                my $fn = $1;
+                next if $fn =~ /\.\./;
+                next if $fn =~ /\//;
+                register $fn;
+            }
+
+            close INDEX;
+
+            unlink $tmpfile;
+        }
+    }
+}
+
+close CONFFILE;
+
+close KNOWNS;

scripts/nix-push-prebuilts

@@ -0,0 +1,40 @@
+#! /usr/bin/perl -w
+
+my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
+my $etcdir = "$prefix/etc/nix";
+my $exportdir = "$prefix/var/nix/prebuilts/exports";
+my $knowns = "$prefix/var/nix/known-prebuilts";
+
+# For performance, put the known hashes in an associative array.
+my %knowns = ();
+open KNOWNS, "<$knowns";
+while (<KNOWNS>) {
+    next unless /([0-9a-z]{32})/;
+    $knowns{$1} = 1;
+}
+close KNOWNS;
+
+# For each installed package, check whether a prebuilt is known.
+
+open PKGS, "nix listinst|";
+open KNOWNS, ">>$knowns";
+
+while (<PKGS>) {
+    chomp;
+    next unless /([0-9a-z]{32})/;
+    my $pkghash = $1;
+    if (!defined $knowns{$1}) {
+        # No known prebuilt exists for this package; so export it.
+        print "exporting $pkghash...\n";
+        system "nix export '$exportdir' $pkghash";
+        if ($?) { die "`nix export' failed"; }
+        print KNOWNS "$pkghash\n";
+    }
+}
+
+close KNOWNS;
+close PKGS;
+
+# Push the prebuilts to the server. !!! FIXME
+
+system "rsync -av -e ssh '$exportdir' losser:/home/eelco/public_html/nix-prebuilts/";

scripts/prebuilts.conf

@@ -0,0 +1,4 @@
+# A list of URLs or local paths from where we obtain prebuilts.
+/nix/var/nix/prebuilts/imports
+/nix/var/nix/prebuilts/exports
+http://losser.st-lab.cs.uu.nl/~eelco/nix-prebuilts/

src/Makefile.am

@@ -13,5 +13,8 @@ install-data-local:
 	$(INSTALL) -d $(localstatedir)/nix/descriptors
 	$(INSTALL) -d $(localstatedir)/nix/sources
 	$(INSTALL) -d $(localstatedir)/nix/links
+	$(INSTALL) -d $(localstatedir)/nix/prebuilts
+	$(INSTALL) -d $(localstatedir)/nix/prebuilts/imports
+	$(INSTALL) -d $(localstatedir)/nix/prebuilts/exports
 	$(INSTALL) -d $(prefix)/pkg
 	$(bindir)/nix init

src/fix.cc

@@ -13,7 +13,6 @@ extern "C" {
 
 
 static string nixDescriptorDir;
-static string nixSourcesDir;
 
 
 static bool verbose = false;
@@ -33,46 +32,6 @@ void registerFile(string filename)
         throw Error("cannot register " + filename + " with Nix");
 }
 
-
-/* Return the directory part of the given path, i.e., everything
-   before the final `/'. */
-string dirOf(string s)
-{
-    unsigned int pos = s.rfind('/');
-    if (pos == string::npos) throw Error("invalid file name");
-    return string(s, 0, pos);
-}
-
-
-/* Return the base name of the given path, i.e., everything following
-   the final `/'. */
-string baseNameOf(string s)
-{
-    unsigned int pos = s.rfind('/');
-    if (pos == string::npos) throw Error("invalid file name");
-    return string(s, pos + 1);
-}
-
-
-/* Download object referenced by the given URL into the sources
-   directory.  Return the file name it was downloaded to. */
-string fetchURL(string url)
-{
-    string filename = baseNameOf(url);
-    string fullname = nixSourcesDir + "/" + filename;
-    struct stat st;
-    if (stat(fullname.c_str(), &st)) {
-        /* !!! quoting */
-        string shellCmd =
-            "cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\"";
-        int res = system(shellCmd.c_str());
-        if (WEXITSTATUS(res) != 0)
-            throw Error("cannot fetch " + url);
-    }
-    return fullname;
-}
-
-
 Error badTerm(const string & msg, ATerm e)
 {
     char * s = ATwriteToString(e);
@@ -120,7 +79,7 @@ bool evaluateBool(ATerm e, EvalContext ctx)
 ATerm evaluate(ATerm e, EvalContext ctx)
 {
     char * s;
-    ATerm e2;
+    ATerm e2, e3;
     ATerm eCond, eTrue, eFalse;
 
     /* Check for normal forms first. */
@@ -166,6 +125,7 @@ ATerm evaluate(ATerm e, EvalContext ctx)
             instantiateDescriptor(filename, ctx).c_str());
     }
 
+#if 0
     /* `Source' copies the specified file to nixSourcesDir, registers
        it with Nix, and returns the hash of the file. */
     else if (ATmatch(e, "Source(<term>)", &e2)) {
@@ -185,15 +145,29 @@ ATerm evaluate(ATerm e, EvalContext ctx)
         registerFile(target);
         return ATmake("File(<str>)", hashFile(target).c_str());
     }
+#endif
 
-    /* `Url' fetches a file from the network, caching it in
+    /* `Local' registers a file with Nix, and returns the file's
-       nixSourcesDir and returning the file name. */
+       hash. */
-    else if (ATmatch(e, "Url(<term>)", &e2)) {
+    else if (ATmatch(e, "Local(<term>)", &e2)) {
-        string url = evaluateStr(e2, ctx);
+        string filename = absPath(evaluateStr(e2, ctx), ctx.dir); /* !!! */
+        string hash = hashFile(filename);
+        return ATmake("File(<str>)", hash.c_str());
+    }
+
+    /* `Url' registers a mapping from a hash to an url with Nix, and
+       returns the hash. */
+    else if (ATmatch(e, "Url(<term>, <term>)", &e2, &e3)) {
+        string hash = evaluateStr(e2, ctx);
+        checkHash(hash);
+        string url = evaluateStr(e3, ctx);
+#if 0
         if (verbose)
             cerr << "fetching " << url << endl;
         string filename = fetchURL(url);
-        return ATmake("Str(<str>)", filename.c_str());
+#endif
+        /* !!! register */
+        return ATmake("File(<str>)", hash.c_str());
     }
 
     /* `If' provides conditional evaluation. */
@@ -329,7 +303,6 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
     if (homeDir) nixHomeDir = homeDir;
 
     nixDescriptorDir = nixHomeDir + "/var/nix/descriptors";
-    nixSourcesDir = nixHomeDir + "/var/nix/sources";
 
     for ( ; argCur != argEnd; argCur++) {
         string arg(*argCur);

src/nix.cc

@@ -28,6 +28,9 @@ static string dbInstPkgs = "pkginst";
 static string dbPrebuilts = "prebuilts";
 
 
+static string nixSourcesDir;
+
+
 /* Wrapper classes that ensures that the database is closed upon
    object destruction. */
 class Db2 : public Db 
@@ -435,9 +438,9 @@ void exportPkgs(string outDir,
         string pkgDir = getPkg(hash);
         string tmpFile = outDir + "/export_tmp";
 
-        string cmd = "cd " + pkgDir + " && tar cvfj " + tmpFile + " .";
+        string cmd = "cd " + pkgDir + " && tar cfj " + tmpFile + " .";
         int res = system(cmd.c_str()); // !!! escaping
-        if (WEXITSTATUS(res) != 0)
+        if (!WIFEXITED(res) || WEXITSTATUS(res) != 0)
             throw Error("cannot tar " + pkgDir);
 
         string prebuiltHash = hashFile(tmpFile);
@@ -458,10 +461,12 @@ void regPrebuilt(string pkgHash, string prebuiltHash)
 }
 
 
-void registerFile(string filename)
+string registerFile(string filename)
 {
     filename = absPath(filename);
-    setDB(dbRefs, hashFile(filename), filename);
+    string hash = hashFile(filename);
+    setDB(dbRefs, hash, filename);
+    return hash;
 }
 
 
@@ -618,8 +623,46 @@ void printGraph(Strings::iterator first, Strings::iterator last)
 }
 
 
-void run(Strings args)
+/* Download object referenced by the given URL into the sources
+   directory.  Return the file name it was downloaded to. */
+string fetchURL(string url)
 {
+    string filename = baseNameOf(url);
+    string fullname = nixSourcesDir + "/" + filename;
+    struct stat st;
+    if (stat(fullname.c_str(), &st)) {
+        /* !!! quoting */
+        string shellCmd =
+            "cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\"";
+        int res = system(shellCmd.c_str());
+        if (WEXITSTATUS(res) != 0)
+            throw Error("cannot fetch " + url);
+    }
+    return fullname;
+}
+
+
+void fetch(string id)
+{
+    string fn;
+
+    /* Fetch the object referenced by id. */
+    if (isHash(id)) {
+        throw Error("not implemented");
+    } else {
+        fn = fetchURL(id);
+    }
+
+    /* Register it by hash. */
+    string hash = registerFile(fn);
+    cout << hash << endl;
+}
+
+
+void fetch(Strings::iterator first, Strings::iterator last)
+{
+    for (Strings::iterator it = first; it != last; it++)
+        fetch(*it);
 }
 
 
@@ -675,6 +718,11 @@ Subcommands:
 
   graph HASH...
     Like closure, but print a dot graph specification.
+
+  fetch ID...  
+    Fetch the objects identified by ID and place them in the Nix
+    sources directory.  ID can be a hash or URL.  Print out the hash
+    of the object.
 ";
 }
 
@@ -686,6 +734,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
     char * homeDir = getenv(nixHomeDirEnvVar.c_str());
     if (homeDir) nixHomeDir = homeDir;
 
+    nixSourcesDir = nixHomeDir + "/var/nix/sources";
+
     /* Parse the global flags. */
     for ( ; argCur != argEnd; argCur++) {
         string arg(*argCur);
@@ -742,6 +792,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
         printClosure(argCur, argEnd);
     } else if (cmd == "graph") {
         printGraph(argCur, argEnd);
+    } else if (cmd == "fetch") {
+        fetch(argCur, argEnd);
     } else
         throw UsageError("unknown command: " + string(cmd));
 }

src/util.hh

@@ -85,17 +85,22 @@ string printHash(unsigned char * buf)
 
     
 /* Verify that a reference is valid (that is, is a MD5 hash code). */
-void checkHash(const string & s)
+bool isHash(const string & s)
 {
-    string err = "invalid reference: " + s;
+    if (s.length() != 32) return false;
-    if (s.length() != 32)
-        throw BadRefError(err);
     for (int i = 0; i < 32; i++) {
         char c = s[i];
         if (!((c >= '0' && c <= '9') ||
               (c >= 'a' && c <= 'f')))
-            throw BadRefError(err);
+            return false;
     }
+    return true;
+}
+
+
+void checkHash(const string & s)
+{
+    if (!isHash(s)) throw BadRefError("invalid reference: " + s);
 }
 
 
@@ -113,4 +118,25 @@ string hashFile(string filename)
 }
 
 
+
+/* Return the directory part of the given path, i.e., everything
+   before the final `/'. */
+string dirOf(string s)
+{
+    unsigned int pos = s.rfind('/');
+    if (pos == string::npos) throw Error("invalid file name");
+    return string(s, 0, pos);
+}
+
+
+/* Return the base name of the given path, i.e., everything following
+   the final `/'. */
+string baseNameOf(string s)
+{
+    unsigned int pos = s.rfind('/');
+    if (pos == string::npos) throw Error("invalid file name");
+    return string(s, pos + 1);
+}
+
+
 #endif /* !__UTIL_H */