From a7a43adb79393084a27589bc929e5a22877ba944 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jul 2012 18:25:01 -0400 Subject: [PATCH 1/8] builtins.storePath: resolve symlinks Needed for Charon/Hydra interaction. --- src/libexpr/primops.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7587dccea..5d5f0bfb3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -502,7 +502,11 @@ static void prim_toPath(EvalState & state, Value * * args, Value & v) static void prim_storePath(EvalState & state, Value * * args, Value & v) { PathSet context; - Path path = canonPath(state.coerceToPath(*args[0], context)); + Path path = state.coerceToPath(*args[0], context); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink + directly in the store. The latter condition is necessary so + e.g. nix-push does the right thing. */ + if (!isStorePath(path)) path = canonPath(path, true); if (!isInStore(path)) throw EvalError(format("path `%1%' is not in the Nix store") % path); Path path2 = toStorePath(path); From 53b24f351852498c52377c2f011617af04bc76fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Jul 2012 09:40:12 -0400 Subject: [PATCH 2/8] Allow disabling log compression --- doc/manual/conf-file.xml | 10 ++++++++++ src/libstore/build.cc | 37 +++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 7fd104eb4..1b19e56b5 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -297,6 +297,16 @@ build-use-chroot = /dev /proc /bin + build-compress-log + + If set to true (the default), + build logs written to /nix/var/log/nix/drvs + will be compressed on the fly using bzip2. Otherwise, they will + not be compressed. + + + + system This option specifies the canonical Nix system diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d5bbd540b..8eb5dfa41 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -769,6 +769,7 @@ private: /* File descriptor for the log file. */ FILE * fLogFile; BZFILE * bzLogFile; + AutoCloseFD fdLogFile; /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -2119,20 +2120,29 @@ Path DerivationGoal::openLogFile() Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str(); createDirs(dir); - Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str(); - AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fd == -1) - throw SysError(format("creating log file `%1%'") % logFileName); - closeOnExec(fd); + if (queryBoolSetting("build-compress-log", true)) { - if (!(fLogFile = fdopen(fd.borrow(), "w"))) - throw SysError(format("opening file `%1%'") % logFileName); + Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str(); + AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName); + closeOnExec(fd); - int err; - if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0))) - throw Error(format("cannot open compressed log file `%1%'") % logFileName); + if (!(fLogFile = fdopen(fd.borrow(), "w"))) + throw SysError(format("opening file `%1%'") % logFileName); - return logFileName; + int err; + if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0))) + throw Error(format("cannot open compressed log file `%1%'") % logFileName); + + return logFileName; + + } else { + Path logFileName = (format("%1%/%2%") % dir % baseNameOf(drvPath)).str(); + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); + closeOnExec(fdLogFile); + return logFileName; + } } @@ -2149,6 +2159,8 @@ void DerivationGoal::closeLogFile() fclose(fLogFile); fLogFile = 0; } + + fdLogFile.close(); } @@ -2180,7 +2192,8 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) int err; BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); - } + } else if (fdLogFile != -1) + writeFull(fdLogFile, (unsigned char *) data.data(), data.size()); } if (hook && fd == hook->fromHook.readSide) From 6c01fb4d68a80f63c692492bb91c1aa2e17b5a8f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Jul 2012 10:06:20 -0400 Subject: [PATCH 3/8] Update Nix 1.1 release notes --- doc/manual/release-notes.xml | 75 ++++++++++++++++++++++++++++++++---- src/libstore/build.cc | 3 ++ 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index 66ced1c9c..69ab0874b 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -15,25 +15,84 @@ - Builds in chroots are now executed in a private network - namespace, meaning that they do not see any network interfaces - except a private loopback interface. This ensures that builds - cannot communicate with processes outside of the chroot, or clash - with other builds by listening on an identical port number. + On Linux, when doing a chroot build, Nix now uses various + namespace features provided by the Linux kernel to improve + build isolation. Namely: + + The private network namespace ensures that + builders cannot talk to the outside world (or vice versa): each + build only sees a private loopback interface. This also means + that two concurrent builds can listen on the same port (e.g. as + part of a test) without conflicting with each + other. + The PID namespace causes each build to start as + PID 1. Processes outside of the chroot are not visible to those + on the inside. On the other hand, processes inside the chroot + are visible from the outside (though with + different PIDs). + The IPC namespace prevents the builder from + communicating with outside processes using SysV IPC mechanisms + (shared memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + The UTS namespace ensures that builders see a + hostname of localhost rather than the actual + hostname. + The private mount namespace was already used by + Nix to ensure that the bind-mounts used to set up the chroot are + cleaned up automatically. + + Build logs are now compressed using bzip2. The command nix-store - -l decompresses them on the fly. + -l decompresses them on the fly. This can be disabled + by setting the option build-compress-log to + false. The creation of build logs in /nix/var/log/nix/drvs can be disabled by setting the new option build-keep-log to - false. - + false. This is useful, for instance, for Hydra + build machines. + + + + Nix now reserves some space in + /nix/var/nix/db/reserved to ensure that the + garbage collector can run successfully if the disk is full. This + is necessary because SQLite transactions fail if the disk is + full. + + + + Added a basic fetchurl function. This + is not intended to replace the fetchurl in + Nixpkgs, but is useful for bootstrapping; e.g., it will allow us + to get rid of the bootstrap binaries in the Nixpkgs source tree + and download them instead. You can use it by doing + import <nix/fetchurl.nix> { url = + url; sha256 = + "hash"; }. (Shea Levy) + + + + Improved RPM spec file. (Michel Alexandre Salim) + + + + Support for on-demand socket-based activation in the Nix + daemon with systemd. + + + + Added a manpage for + nix.conf5. + diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8eb5dfa41..12940e268 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1796,6 +1796,9 @@ void DerivationGoal::startBuilder() with outside processes using SysV IPC mechanisms (shared memory, message queues, semaphores). It also ensures that all IPC objects are destroyed when the builder exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. */ #if CHROOT_ENABLED if (useChroot) { From 51d71ad3d7527596dc22d6dd9e9e70f2cd9faea9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Jul 2012 11:49:47 -0400 Subject: [PATCH 4/8] Manual: Don't claim we support Cygwin --- doc/manual/introduction.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/introduction.xml b/doc/manual/introduction.xml index 5641d110b..21b1df156 100644 --- a/doc/manual/introduction.xml +++ b/doc/manual/introduction.xml @@ -225,7 +225,7 @@ href="docs/papers.html#servicecm">SCM-12 paper. Portability Nix should run on most Unix systems, including Linux, FreeBSD and -Mac OS X. It is also supported on Windows using Cygwin. +Mac OS X. From 1217204c81b0b6f02df99adfc8414a181299535c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Jul 2012 14:07:52 -0400 Subject: [PATCH 5/8] Remove dead code --- src/libexpr/eval.cc | 9 --------- src/libexpr/eval.hh | 4 ---- 2 files changed, 13 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6ce65e3e1..cf7c62ad2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -143,9 +143,7 @@ EvalState::EvalState() , staticBaseEnv(false, 0) { nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0; - nrEvaluated = recursionDepth = maxRecursionDepth = 0; nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0; - deepestStack = (char *) -1; #if HAVE_BOEHMGC static bool gcInitialised = true; @@ -190,7 +188,6 @@ EvalState::EvalState() EvalState::~EvalState() { - assert(recursionDepth == 0); } @@ -1206,12 +1203,6 @@ void EvalState::printStats() printMsg(v, format(" time elapsed: %1%") % cpuTime); printMsg(v, format(" size of a value: %1%") % sizeof(Value)); - printMsg(v, format(" expressions evaluated: %1%") % nrEvaluated); - char x; - printMsg(v, format(" stack space used: %1% bytes") % (&x - deepestStack)); - printMsg(v, format(" max eval() nesting depth: %1%") % maxRecursionDepth); - printMsg(v, format(" stack space per eval() level: %1% bytes") - % ((&x - deepestStack) / (float) maxRecursionDepth)); printMsg(v, format(" environments allocated: %1% (%2% bytes)") % nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *))); printMsg(v, format(" list elements: %1% (%2% bytes)") diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index c4ba170e8..bab9303b0 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -242,13 +242,9 @@ private: unsigned long nrValuesInEnvs; unsigned long nrValues; unsigned long nrListElems; - unsigned long nrEvaluated; unsigned long nrAttrsets; unsigned long nrOpUpdates; unsigned long nrOpUpdateValuesCopied; - unsigned int recursionDepth; - unsigned int maxRecursionDepth; - char * deepestStack; /* for measuring stack usage */ friend class RecursionCounter; friend class ExprOpUpdate; From 3a9fdf2747bc7436fc3c1fd5f9accd5675d4295e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Jul 2012 15:55:30 -0400 Subject: [PATCH 6/8] Return an exit code of 100 for cached failed builds Exit code 100 should be returned for all permanent failures. This includes cached failures. Fixes #34. --- src/libstore/build.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 12940e268..26268f6dd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2234,6 +2234,7 @@ bool DerivationGoal::pathFailed(const Path & path) if (printBuildTrace) printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path); + worker.permanentFailure = true; amDone(ecFailed); return true; From 220818f758d2facc194f567f35ca677ef79393bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Jul 2012 16:55:45 -0400 Subject: [PATCH 7/8] queryPathInfo(): return hash in base-32 if desired Cherry-picked from the no-manifests branch. --- perl/lib/Nix/Store.xs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 8ca72b62a..2ebff5575 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -86,7 +86,7 @@ SV * queryDeriver(char * path) } -SV * queryPathInfo(char * path) +SV * queryPathInfo(char * path, int base32) PPCODE: try { doInit(); @@ -95,7 +95,7 @@ SV * queryPathInfo(char * path) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(info.deriver.c_str(), 0))); - string s = "sha256:" + printHash(info.hash); + string s = "sha256:" + (base32 ? printHash32(info.hash) : printHash(info.hash)); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info.registrationTime); mXPUSHi(info.narSize); From ccc52adfb2121ade510d35dc9b91193af9fa731e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Jul 2012 18:55:39 -0400 Subject: [PATCH 8/8] Add function queryPathFromHashPart() To implement binary caches efficiently, Hydra needs to be able to map the hash part of a store path (e.g. "gbg...zr7") to the full store path (e.g. "/nix/store/gbg...kzr7-subversion-1.7.5"). (The binary cache mechanism uses hash parts as a key for looking up store paths to ensure privacy.) However, doing a search in the Nix store for /nix/store/* is expensive since it requires reading the entire directory. queryPathFromHashPart() prevents this by doing a cheap database lookup. --- perl/lib/Nix/Store.pm | 1 + perl/lib/Nix/Store.xs | 11 +++++++++++ src/libstore/local-store.cc | 24 ++++++++++++++++++++++++ src/libstore/local-store.hh | 3 +++ src/libstore/remote-store.cc | 12 ++++++++++++ src/libstore/remote-store.hh | 2 ++ src/libstore/store-api.hh | 4 ++++ src/libstore/worker-protocol.hh | 1 + src/nix-worker/nix-worker.cc | 9 +++++++++ 9 files changed, 67 insertions(+) diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index 8312a732c..2e79c74fe 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -14,6 +14,7 @@ our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); our @EXPORT = qw( isValidPath queryReferences queryPathInfo queryDeriver queryPathHash + queryPathFromHashPart topoSortPaths computeFSClosure followLinksToStorePath exportPaths hashPath hashFile hashString addToStore makeFixedOutputPath diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 2ebff5575..76de674e6 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -108,6 +108,17 @@ SV * queryPathInfo(char * path, int base32) } +SV * queryPathFromHashPart(char * hashPart) + PPCODE: + try { + doInit(); + Path path = store->queryPathFromHashPart(hashPart); + XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); + } catch (Error & e) { + croak(e.what()); + } + + SV * computeFSClosure(int flipDirection, int includeOutputs, ...) PPCODE: try { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1ce62aeaf..30398a244 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -405,6 +405,10 @@ void LocalStore::openDB(bool create) "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); stmtQueryDerivationOutputs.create(db, "select id, path from DerivationOutputs where drv = ?;"); + // Use "path >= ?" with limit 1 rather than "path like '?%'" to + // ensure efficient lookup. + stmtQueryPathFromHashPart.create(db, + "select path from ValidPaths where path >= ? limit 1;"); } @@ -865,6 +869,26 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path) } +Path LocalStore::queryPathFromHashPart(const string & hashPart) +{ + if (hashPart.size() != 32) throw Error("invalid hash part"); + + SQLiteTxn txn(db); + + Path prefix = nixStore + "/" + hashPart; + + SQLiteStmtUse use(stmtQueryPathFromHashPart); + stmtQueryPathFromHashPart.bind(prefix); + + int res = sqlite3_step(stmtQueryPathFromHashPart); + if (res == SQLITE_DONE) return ""; + if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + + const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; +} + + void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) { if (run.pid != -1) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index aa8e8582f..65ee029c2 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -121,6 +121,8 @@ public: StringSet queryDerivationOutputNames(const Path & path); + Path queryPathFromHashPart(const string & hashPart); + PathSet querySubstitutablePaths(); bool hasSubstitutes(const Path & path); @@ -217,6 +219,7 @@ private: SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; int getSchema(); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5e5561a6a..cbb70b2fd 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -341,6 +341,18 @@ PathSet RemoteStore::queryDerivationOutputNames(const Path & path) } +Path RemoteStore::queryPathFromHashPart(const string & hashPart) +{ + openConnection(); + writeInt(wopQueryPathFromHashPart, to); + writeString(hashPart, to); + processStderr(); + Path path = readString(from); + if (!path.empty()) assertStorePath(path); + return path; +} + + Path RemoteStore::addToStore(const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index e9f40da6d..f0e5dbf76 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -43,6 +43,8 @@ public: StringSet queryDerivationOutputNames(const Path & path); + Path queryPathFromHashPart(const string & hashPart); + bool hasSubstitutes(const Path & path); bool querySubstitutablePathInfo(const Path & path, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index bf3269f57..0ab15c380 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -139,6 +139,10 @@ public: /* Query the output names of the derivation denoted by `path'. */ virtual StringSet queryDerivationOutputNames(const Path & path) = 0; + + /* Query the full store path given the hash part of a valid store + path, or "" if the path doesn't exist. */ + virtual Path queryPathFromHashPart(const string & hashPart) = 0; /* Query whether a path has substitutes. */ virtual bool hasSubstitutes(const Path & path) = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 6a5f0ed40..b08410fa1 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -40,6 +40,7 @@ typedef enum { wopQueryPathInfo = 26, wopImportPaths = 27, wopQueryDerivationOutputNames = 28, + wopQueryPathFromHashPart = 29, } WorkerOp; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 2f0a2ab20..74a619c71 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -350,6 +350,15 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryPathFromHashPart: { + string hashPart = readString(from); + startWork(); + Path path = store->queryPathFromHashPart(hashPart); + stopWork(); + writeString(path, to); + break; + } + case wopAddToStore: { string baseName = readString(from); bool fixed = readInt(from) == 1; /* obsolete */