Merge branch 'master' into no-manifests

This commit is contained in:
Eelco Dolstra 2012-07-18 10:47:59 -04:00
commit fe241ece29
16 changed files with 184 additions and 40 deletions

View file

@ -297,6 +297,16 @@ build-use-chroot = /dev /proc /bin</programlisting>
</varlistentry>
<varlistentry><term><literal>build-compress-log</literal></term>
<listitem><para>If set to <literal>true</literal> (the default),
build logs written to <filename>/nix/var/log/nix/drvs</filename>
will be compressed on the fly using bzip2. Otherwise, they will
not be compressed.</para></listitem>
</varlistentry>
<varlistentry><term><literal>binary-caches</literal></term>
<listitem><para>A list of URLs of binary caches, separated by

View file

@ -225,7 +225,7 @@ href="docs/papers.html#servicecm">SCM-12 paper</a>.</para>
<simplesect><title>Portability</title>
<para>Nix should run on most Unix systems, including Linux, FreeBSD and
Mac OS X. It is also supported on Windows using Cygwin.</para>
Mac OS X.<!-- It is also supported on Windows using Cygwin.--></para>
</simplesect>

View file

@ -15,24 +15,83 @@
<itemizedlist>
<listitem>
<para>Builds in chroots are now executed in a private network
namespace, meaning that they do not see any network interfaces
except a private loopback interface. This ensures that builds
cannot communicate with processes outside of the chroot, or clash
with other builds by listening on an identical port number.</para>
<para>On Linux, when doing a chroot build, Nix now uses various
namespace features provided by the Linux kernel to improve
build isolation. Namely:
<itemizedlist>
<listitem><para>The private network namespace ensures that
builders cannot talk to the outside world (or vice versa): each
build only sees a private loopback interface. This also means
that two concurrent builds can listen on the same port (e.g. as
part of a test) without conflicting with each
other.</para></listitem>
<listitem><para>The PID namespace causes each build to start as
PID 1. Processes outside of the chroot are not visible to those
on the inside. On the other hand, processes inside the chroot
<emphasis>are</emphasis> visible from the outside (though with
different PIDs).</para></listitem>
<listitem><para>The IPC namespace prevents the builder from
communicating with outside processes using SysV IPC mechanisms
(shared memory, message queues, semaphores). It also ensures
that all IPC objects are destroyed when the builder
exits.</para></listitem>
<listitem><para>The UTS namespace ensures that builders see a
hostname of <literal>localhost</literal> rather than the actual
hostname.</para></listitem>
<listitem><para>The private mount namespace was already used by
Nix to ensure that the bind-mounts used to set up the chroot are
cleaned up automatically.</para></listitem>
</itemizedlist>
</para>
</listitem>
<listitem>
<para>Build logs are now compressed using
<command>bzip2</command>. The command <command>nix-store
-l</command> decompresses them on the fly.</para>
-l</command> decompresses them on the fly. This can be disabled
by setting the option <literal>build-compress-log</literal> to
<literal>false</literal>.</para>
</listitem>
<listitem>
<para>The creation of build logs in
<filename>/nix/var/log/nix/drvs</filename> can be disabled by
setting the new option <literal>build-keep-log</literal> to
<literal>false</literal>.</para>
<literal>false</literal>. This is useful, for instance, for Hydra
build machines.</para>
</listitem>
<listitem>
<para>Nix now reserves some space in
<filename>/nix/var/nix/db/reserved</filename> to ensure that the
garbage collector can run successfully if the disk is full. This
is necessary because SQLite transactions fail if the disk is
full.</para>
</listitem>
<listitem>
<para>Added a basic <function>fetchurl</function> function. This
is not intended to replace the <function>fetchurl</function> in
Nixpkgs, but is useful for bootstrapping; e.g., it will allow us
to get rid of the bootstrap binaries in the Nixpkgs source tree
and download them instead. You can use it by doing
<literal>import &lt;nix/fetchurl.nix> { url =
<replaceable>url</replaceable>; sha256 =
"<replaceable>hash</replaceable>"; }</literal>. (Shea Levy)</para>
</listitem>
<listitem>
<para>Improved RPM spec file. (Michel Alexandre Salim)</para>
</listitem>
<listitem>
<para>Support for on-demand socket-based activation in the Nix
daemon with <command>systemd</command>.</para>
</listitem>
<listitem>
<para>Added a manpage for
<citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
</listitem>
<listitem>

View file

@ -14,6 +14,7 @@ our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
our @EXPORT = qw(
isValidPath queryReferences queryPathInfo queryDeriver queryPathHash
queryPathFromHashPart
topoSortPaths computeFSClosure followLinksToStorePath exportPaths
hashPath hashFile hashString
addToStore makeFixedOutputPath

View file

@ -108,6 +108,17 @@ SV * queryPathInfo(char * path, int base32)
}
SV * queryPathFromHashPart(char * hashPart)
PPCODE:
try {
doInit();
Path path = store->queryPathFromHashPart(hashPart);
XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
} catch (Error & e) {
croak(e.what());
}
SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
PPCODE:
try {

View file

@ -143,9 +143,7 @@ EvalState::EvalState()
, staticBaseEnv(false, 0)
{
nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0;
nrEvaluated = recursionDepth = maxRecursionDepth = 0;
nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0;
deepestStack = (char *) -1;
#if HAVE_BOEHMGC
static bool gcInitialised = true;
@ -190,7 +188,6 @@ EvalState::EvalState()
EvalState::~EvalState()
{
assert(recursionDepth == 0);
}
@ -1206,12 +1203,6 @@ void EvalState::printStats()
printMsg(v, format(" time elapsed: %1%") % cpuTime);
printMsg(v, format(" size of a value: %1%") % sizeof(Value));
printMsg(v, format(" expressions evaluated: %1%") % nrEvaluated);
char x;
printMsg(v, format(" stack space used: %1% bytes") % (&x - deepestStack));
printMsg(v, format(" max eval() nesting depth: %1%") % maxRecursionDepth);
printMsg(v, format(" stack space per eval() level: %1% bytes")
% ((&x - deepestStack) / (float) maxRecursionDepth));
printMsg(v, format(" environments allocated: %1% (%2% bytes)")
% nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *)));
printMsg(v, format(" list elements: %1% (%2% bytes)")

View file

@ -242,13 +242,9 @@ private:
unsigned long nrValuesInEnvs;
unsigned long nrValues;
unsigned long nrListElems;
unsigned long nrEvaluated;
unsigned long nrAttrsets;
unsigned long nrOpUpdates;
unsigned long nrOpUpdateValuesCopied;
unsigned int recursionDepth;
unsigned int maxRecursionDepth;
char * deepestStack; /* for measuring stack usage */
friend class RecursionCounter;
friend class ExprOpUpdate;

View file

@ -508,7 +508,11 @@ static void prim_toPath(EvalState & state, Value * * args, Value & v)
static void prim_storePath(EvalState & state, Value * * args, Value & v)
{
PathSet context;
Path path = canonPath(state.coerceToPath(*args[0], context));
Path path = state.coerceToPath(*args[0], context);
/* Resolve symlinks in path, unless path itself is a symlink
directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */
if (!isStorePath(path)) path = canonPath(path, true);
if (!isInStore(path))
throw EvalError(format("path `%1%' is not in the Nix store") % path);
Path path2 = toStorePath(path);

View file

@ -777,6 +777,7 @@ private:
/* File descriptor for the log file. */
FILE * fLogFile;
BZFILE * bzLogFile;
AutoCloseFD fdLogFile;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
@ -1801,6 +1802,9 @@ void DerivationGoal::startBuilder()
with outside processes using SysV IPC mechanisms (shared
memory, message queues, semaphores). It also ensures that
all IPC objects are destroyed when the builder exits.
- The UTS namespace ensures that builders see a hostname of
localhost rather than the actual hostname.
*/
#if CHROOT_ENABLED
if (useChroot) {
@ -2125,20 +2129,29 @@ Path DerivationGoal::openLogFile()
Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str();
createDirs(dir);
Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
if (fd == -1)
throw SysError(format("creating log file `%1%'") % logFileName);
closeOnExec(fd);
if (queryBoolSetting("build-compress-log", true)) {
if (!(fLogFile = fdopen(fd.borrow(), "w")))
throw SysError(format("opening file `%1%'") % logFileName);
Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
closeOnExec(fd);
int err;
if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0)))
throw Error(format("cannot open compressed log file `%1%'") % logFileName);
if (!(fLogFile = fdopen(fd.borrow(), "w")))
throw SysError(format("opening file `%1%'") % logFileName);
return logFileName;
int err;
if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0)))
throw Error(format("cannot open compressed log file `%1%'") % logFileName);
return logFileName;
} else {
Path logFileName = (format("%1%/%2%") % dir % baseNameOf(drvPath)).str();
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
closeOnExec(fdLogFile);
return logFileName;
}
}
@ -2155,6 +2168,8 @@ void DerivationGoal::closeLogFile()
fclose(fLogFile);
fLogFile = 0;
}
fdLogFile.close();
}
@ -2186,7 +2201,8 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
int err;
BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err);
}
} else if (fdLogFile != -1)
writeFull(fdLogFile, (unsigned char *) data.data(), data.size());
}
if (hook && fd == hook->fromHook.readSide)
@ -2224,6 +2240,7 @@ bool DerivationGoal::pathFailed(const Path & path)
if (printBuildTrace)
printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
worker.permanentFailure = true;
amDone(ecFailed);
return true;

View file

@ -405,6 +405,10 @@ void LocalStore::openDB(bool create)
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
stmtQueryDerivationOutputs.create(db,
"select id, path from DerivationOutputs where drv = ?;");
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
// ensure efficient lookup.
stmtQueryPathFromHashPart.create(db,
"select path from ValidPaths where path >= ? limit 1;");
}
@ -874,6 +878,26 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path)
}
Path LocalStore::queryPathFromHashPart(const string & hashPart)
{
if (hashPart.size() != 32) throw Error("invalid hash part");
SQLiteTxn txn(db);
Path prefix = nixStore + "/" + hashPart;
SQLiteStmtUse use(stmtQueryPathFromHashPart);
stmtQueryPathFromHashPart.bind(prefix);
int res = sqlite3_step(stmtQueryPathFromHashPart);
if (res == SQLITE_DONE) return "";
if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
}
void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
{
if (run.pid != -1) return;

View file

@ -123,6 +123,8 @@ public:
StringSet queryDerivationOutputNames(const Path & path);
Path queryPathFromHashPart(const string & hashPart);
PathSet querySubstitutablePaths(const PathSet & paths);
void querySubstitutablePathInfos(const Path & substituter,
@ -217,6 +219,7 @@ private:
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
int getSchema();

View file

@ -395,6 +395,18 @@ PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
}
Path RemoteStore::queryPathFromHashPart(const string & hashPart)
{
openConnection();
writeInt(wopQueryPathFromHashPart, to);
writeString(hashPart, to);
processStderr();
Path path = readString(from);
if (!path.empty()) assertStorePath(path);
return path;
}
Path RemoteStore::addToStore(const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter)
{

View file

@ -45,6 +45,8 @@ public:
StringSet queryDerivationOutputNames(const Path & path);
Path queryPathFromHashPart(const string & hashPart);
PathSet querySubstitutablePaths(const PathSet & paths);
void querySubstitutablePathInfos(const PathSet & paths,

View file

@ -145,6 +145,10 @@ public:
/* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
/* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
/* Query which of the given paths have substitutes. */
virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;

View file

@ -40,9 +40,10 @@ typedef enum {
wopQueryPathInfo = 26,
wopImportPaths = 27,
wopQueryDerivationOutputNames = 28,
wopQuerySubstitutablePathInfos = 29,
wopQueryValidPaths = 30,
wopQuerySubstitutablePaths = 31,
wopQueryPathFromHashPart = 29,
wopQuerySubstitutablePathInfos = 30,
wopQueryValidPaths = 31,
wopQuerySubstitutablePaths = 32,
} WorkerOp;

View file

@ -368,6 +368,15 @@ static void performOp(unsigned int clientVersion,
break;
}
case wopQueryPathFromHashPart: {
string hashPart = readString(from);
startWork();
Path path = store->queryPathFromHashPart(hashPart);
stopWork();
writeString(path, to);
break;
}
case wopAddToStore: {
string baseName = readString(from);
bool fixed = readInt(from) == 1; /* obsolete */