From eaaa13ce475687ac0d0e2d3c22c7de73a9a83a49 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 13:14:51 +0000 Subject: [PATCH 001/128] * Experimental branch for using SQLite to store the Nix database. From c1a07f94451cfa93aa9ac986188d0e9a536e4b9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 13:16:59 +0000 Subject: [PATCH 002/128] * Convert the Nix database to SQLite. --- configure.ac | 3 + src/bin2c/bin2c.c | 4 +- src/libstore/Makefile.am | 5 ++ src/libstore/local-store.cc | 119 ++++++++++++++++++++++++++++++++---- src/libstore/local-store.hh | 8 ++- src/libstore/schema.sql | 27 ++++++++ 6 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 src/libstore/schema.sql diff --git a/configure.ac b/configure.ac index f6d983fda..f726b1ae4 100644 --- a/configure.ac +++ b/configure.ac @@ -249,6 +249,9 @@ AC_SUBST(bzip2_bin_test) AC_CHECK_LIB(pthread, pthread_mutex_init) +LDFLAGS="-lsqlite3" + + AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state], [do not initialise DB etc. in `make install']), init_state=$enableval, init_state=yes) diff --git a/src/bin2c/bin2c.c b/src/bin2c/bin2c.c index 18bf81d69..5ed8a5708 100644 --- a/src/bin2c/bin2c.c +++ b/src/bin2c/bin2c.c @@ -14,10 +14,10 @@ int main(int argc, char * * argv) { int c; if (argc != 2) abort(); - print("static unsigned char %s[] = {", argv[1]); + print("static unsigned char %s[] = { ", argv[1]); while ((c = getchar()) != EOF) { print("0x%02x, ", (unsigned char) c); } - print("};\n"); + print("0 };\n"); return 0; } diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 863871519..0140857b8 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -19,5 +19,10 @@ EXTRA_DIST = derivations-ast.def derivations-ast.cc AM_CXXFLAGS = -Wall \ -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil +local-store.lo: schema.sql.hh + +%.sql.hh: %.sql + ../bin2c/bin2c schema < $< > $@ || (rm $@ && exit 1) + derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def $(perl) $(srcdir)/../aterm-helper.pl derivations-ast.hh derivations-ast.cc < $(srcdir)/derivations-ast.def diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7c8db745c..55637dd85 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -22,6 +22,16 @@ namespace nix { +class SQLiteError : public Error +{ +public: + SQLiteError(sqlite3 * db, const format & f) + : Error(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)) + { + } +}; + + void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -42,6 +52,7 @@ void checkStoreNotSymlink() LocalStore::LocalStore() { + db = 0; substitutablePathsLoaded = false; schemaPath = nixDBPath + "/schema"; @@ -50,9 +61,6 @@ LocalStore::LocalStore() /* Create missing state directories if they don't already exist. */ createDirs(nixStore); - createDirs(nixDBPath + "/info"); - createDirs(nixDBPath + "/referrer"); - createDirs(nixDBPath + "/failed"); Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); @@ -88,7 +96,12 @@ LocalStore::LocalStore() writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } if (curSchema == 1) throw Error("your Nix store is no longer supported"); - if (curSchema < nixSchemaVersion) upgradeStore12(); + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + if (curSchema < 6) upgradeStore6(); doFsync = queryBoolSetting("fsync-metadata", false); } @@ -99,6 +112,9 @@ LocalStore::~LocalStore() try { flushDelayedUpdates(); + if (db && sqlite3_close(db) != SQLITE_OK) + throw SQLiteError(db, "closing database"); + foreach (RunningSubstituters::iterator, i, runningSubstituters) { i->second.to.close(); i->second.from.close(); @@ -123,6 +139,22 @@ int LocalStore::getSchema() } +#include "schema.sql.hh" + +void LocalStore::initSchema() +{ + if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error("cannot open SQLite database"); + + if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + throw SQLiteError(db, "sett"); + + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "initialising database schema"); +} + + void canonicalisePathMetaData(const Path & path, bool recurse) { checkInterrupt(); @@ -1171,15 +1203,78 @@ void LocalStore::verifyStore(bool checkContents) } -/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12). The - old schema uses Berkeley DB, the new one stores store path - meta-information in files. */ -void LocalStore::upgradeStore12() +/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ +void LocalStore::upgradeStore6() { - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); + if (!lockFile(globalLock, ltWrite, false)) { + printMsg(lvlError, "waiting for exclusive access to the Nix store..."); + lockFile(globalLock, ltWrite, true); + } + + printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + + initSchema(); + + PathSet validPaths = queryValidPaths(); + + sqlite3_stmt * registerStmt; + if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime) values (?, ?, ?);", + -1, ®isterStmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + + sqlite3_stmt * addRefStmt; + if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", + -1, &addRefStmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "running `begin' command"); + + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfo(*i, true); + + if (sqlite3_reset(registerStmt) != SQLITE_OK) + throw SQLiteError(db, "resetting statement"); + if (sqlite3_bind_text(registerStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 1"); + string h = "sha256:" + printHash(info.hash); + if (sqlite3_bind_text(registerStmt, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 2"); + if (sqlite3_bind_int(registerStmt, 3, info.registrationTime) != SQLITE_OK) + throw SQLiteError(db, "binding argument 3"); + if (sqlite3_step(registerStmt) != SQLITE_DONE) + throw SQLiteError(db, "registering valid path in database"); + + foreach (PathSet::iterator, j, info.references) { + if (sqlite3_reset(addRefStmt) != SQLITE_OK) + throw SQLiteError(db, "resetting statement"); + if (sqlite3_bind_text(addRefStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 1"); + if (sqlite3_bind_text(addRefStmt, 2, j->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 2"); + if (sqlite3_step(addRefStmt) != SQLITE_DONE) + throw SQLiteError(db, "adding reference to database"); + } + + std::cerr << "."; + } + + std::cerr << "\n"; + + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "running `commit' command"); + + if (sqlite3_finalize(registerStmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + + if (sqlite3_finalize(addRefStmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + + throw Error("foo"); + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock, ltRead, true); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 31f8d9109..f29565680 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -6,6 +6,8 @@ #include "store-api.hh" #include "util.hh" +#include + namespace nix { @@ -161,8 +163,12 @@ private: /* Whether to do an fsync() after writing Nix metadata. */ bool doFsync; + sqlite3 * db; + int getSchema(); + void initSchema(); + void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false); @@ -177,7 +183,7 @@ private: void invalidatePath(const Path & path); - void upgradeStore12(); + void upgradeStore6(); struct GCState; diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql new file mode 100644 index 000000000..e24132943 --- /dev/null +++ b/src/libstore/schema.sql @@ -0,0 +1,27 @@ +pragma foreign_keys = on; + +create table if not exists ValidPaths ( + path text primary key not null, + hash text not null, + registrationTime integer not null +); + +create table if not exists Refs ( + referrer text not null, + reference text not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(path) + on delete cascade + deferrable initially deferred, + foreign key (reference) references ValidPaths(path) + on delete restrict + deferrable initially deferred +); + +create table if not exists FailedDerivations ( + path text primary key not null, + time integer not null +); + +create index IndexReferrer on Refs(referrer); +create index IndexReference on Refs(reference); From dbddac0fe91072b402ccb3801c952e3159f0cba4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 13:40:46 +0000 Subject: [PATCH 003/128] * Assign an integer id to every row in the ValidPaths table in order to make the Refs table more space-efficient. For instance, this reduces the size of the database on my laptop from 93 MiB to 18 MiB. (It was 72 MiB with the old schema on an ext3 disk with a 1 KiB block size.) --- src/libstore/local-store.cc | 18 ++++++++++++++++-- src/libstore/schema.sql | 21 +++++++++------------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 55637dd85..03c5d76b3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1229,6 +1229,8 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `begin' command"); + + std::map pathToId; foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); @@ -1245,12 +1247,24 @@ void LocalStore::upgradeStore6() if (sqlite3_step(registerStmt) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); + pathToId[*i] = sqlite3_last_insert_rowid(db); + + std::cerr << "."; + } + + std::cerr << "|"; + + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfo(*i, true); + foreach (PathSet::iterator, j, info.references) { if (sqlite3_reset(addRefStmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); - if (sqlite3_bind_text(addRefStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_int(addRefStmt, 1, pathToId[*i]) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); - if (sqlite3_bind_text(addRefStmt, 2, j->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (pathToId.find(*j) == pathToId.end()) + throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); + if (sqlite3_bind_int(addRefStmt, 2, pathToId[*j]) != SQLITE_OK) throw SQLiteError(db, "binding argument 2"); if (sqlite3_step(addRefStmt) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index e24132943..3c07978fc 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -1,27 +1,24 @@ pragma foreign_keys = on; create table if not exists ValidPaths ( - path text primary key not null, + id integer primary key autoincrement not null, + path text unique not null, hash text not null, registrationTime integer not null ); create table if not exists Refs ( - referrer text not null, - reference text not null, + referrer integer not null, + reference integer not null, primary key (referrer, reference), - foreign key (referrer) references ValidPaths(path) - on delete cascade - deferrable initially deferred, - foreign key (reference) references ValidPaths(path) - on delete restrict - deferrable initially deferred + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict ); +create index if not exists IndexReferrer on Refs(referrer); +create index if not exists IndexReference on Refs(reference); + create table if not exists FailedDerivations ( path text primary key not null, time integer not null ); - -create index IndexReferrer on Refs(referrer); -create index IndexReference on Refs(reference); From a053d2d8e53f2967c64ab2b204727e4c27f06c0e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 13:48:18 +0000 Subject: [PATCH 004/128] * Add the deriver to the ValidPaths table. In principle we could now store all the derivers of a path efficiently. But that opens a big can of worms with respect to garbage collector semantics. --- src/libstore/local-store.cc | 33 ++++++++++++++++++++------------- src/libstore/schema.sql | 3 ++- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 03c5d76b3..35391f3aa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1217,19 +1217,14 @@ void LocalStore::upgradeStore6() PathSet validPaths = queryValidPaths(); - sqlite3_stmt * registerStmt; - if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime) values (?, ?, ?);", - -1, ®isterStmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); - - sqlite3_stmt * addRefStmt; - if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", - -1, &addRefStmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `begin' command"); + sqlite3_stmt * registerStmt; + if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);", + -1, ®isterStmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + std::map pathToId; foreach (PathSet::iterator, i, validPaths) { @@ -1244,6 +1239,13 @@ void LocalStore::upgradeStore6() throw SQLiteError(db, "binding argument 2"); if (sqlite3_bind_int(registerStmt, 3, info.registrationTime) != SQLITE_OK) throw SQLiteError(db, "binding argument 3"); + if (info.deriver != "") { + if (sqlite3_bind_text(registerStmt, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 4"); + } else { + if (sqlite3_bind_null(registerStmt, 4) != SQLITE_OK) + throw SQLiteError(db, "binding argument 4"); + } if (sqlite3_step(registerStmt) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); @@ -1252,8 +1254,16 @@ void LocalStore::upgradeStore6() std::cerr << "."; } + if (sqlite3_finalize(registerStmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + std::cerr << "|"; + sqlite3_stmt * addRefStmt; + if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", + -1, &addRefStmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); @@ -1278,9 +1288,6 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `commit' command"); - if (sqlite3_finalize(registerStmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); - if (sqlite3_finalize(addRefStmt) != SQLITE_OK) throw SQLiteError(db, "finalizing statement"); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 3c07978fc..dc53f452c 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -4,7 +4,8 @@ create table if not exists ValidPaths ( id integer primary key autoincrement not null, path text unique not null, hash text not null, - registrationTime integer not null + registrationTime integer not null, + deriver text ); create table if not exists Refs ( From e0305bb7a8b24fe1ea8b36cc2e5fe77ab151ae2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 14:30:42 +0000 Subject: [PATCH 005/128] * Some wrapper objects to ensure that SQLite objects are properly destroyed. --- src/libstore/local-store.cc | 118 ++++++++++++++++++++++-------------- src/libstore/local-store.hh | 40 ++++++++++-- 2 files changed, 109 insertions(+), 49 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 35391f3aa..870644c88 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -18,6 +18,8 @@ #include #include +#include + namespace nix { @@ -32,6 +34,37 @@ public: }; +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throw SQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + this->db = db; +} + + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + + void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -52,7 +85,6 @@ void checkStoreNotSymlink() LocalStore::LocalStore() { - db = 0; substitutablePathsLoaded = false; schemaPath = nixDBPath + "/schema"; @@ -73,6 +105,8 @@ LocalStore::LocalStore() checkStoreNotSymlink(); + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ try { Path globalLockPath = nixDBPath + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); @@ -86,22 +120,34 @@ LocalStore::LocalStore() printMsg(lvlError, "waiting for the big Nix store lock..."); lockFile(globalLock, ltRead, true); } + + /* Open the Nix database. */ + if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error("cannot open SQLite database"); + + if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + throw SQLiteError(db, "setting timeout"); + /* Check the current database schema and if necessary do an + upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") % curSchema % nixSchemaVersion); if (curSchema == 0) { /* new store */ - curSchema = nixSchemaVersion; + curSchema = nixSchemaVersion; + initSchema(); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } - if (curSchema == 1) throw Error("your Nix store is no longer supported"); - if (curSchema < 5) + else if (curSchema == 1) throw Error("your Nix store is no longer supported"); + else if (curSchema < 5) throw Error( "Your Nix store has a database in Berkeley DB format,\n" "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 0.12 first."); - if (curSchema < 6) upgradeStore6(); + else if (curSchema < 6) upgradeStore6(); + else prepareStatements(); doFsync = queryBoolSetting("fsync-metadata", false); } @@ -112,9 +158,6 @@ LocalStore::~LocalStore() try { flushDelayedUpdates(); - if (db && sqlite3_close(db) != SQLITE_OK) - throw SQLiteError(db, "closing database"); - foreach (RunningSubstituters::iterator, i, runningSubstituters) { i->second.to.close(); i->second.from.close(); @@ -143,15 +186,19 @@ int LocalStore::getSchema() void LocalStore::initSchema() { - if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) - throw Error("cannot open SQLite database"); - - if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) - throw SQLiteError(db, "sett"); - if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "initialising database schema"); + + prepareStatements(); +} + + +void LocalStore::prepareStatements() +{ + stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + stmtAddReference.create(db, + "insert into Refs (referrer, reference) values (?, ?);"); } @@ -1220,33 +1267,28 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `begin' command"); - sqlite3_stmt * registerStmt; - if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);", - -1, ®isterStmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); - std::map pathToId; foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); - if (sqlite3_reset(registerStmt) != SQLITE_OK) + if (sqlite3_reset(stmtRegisterValidPath) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); - if (sqlite3_bind_text(registerStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); string h = "sha256:" + printHash(info.hash); - if (sqlite3_bind_text(registerStmt, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 2"); - if (sqlite3_bind_int(registerStmt, 3, info.registrationTime) != SQLITE_OK) + if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) throw SQLiteError(db, "binding argument 3"); if (info.deriver != "") { - if (sqlite3_bind_text(registerStmt, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 4"); } else { - if (sqlite3_bind_null(registerStmt, 4) != SQLITE_OK) + if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) throw SQLiteError(db, "binding argument 4"); } - if (sqlite3_step(registerStmt) != SQLITE_DONE) + if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); pathToId[*i] = sqlite3_last_insert_rowid(db); @@ -1254,29 +1296,21 @@ void LocalStore::upgradeStore6() std::cerr << "."; } - if (sqlite3_finalize(registerStmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); - std::cerr << "|"; - sqlite3_stmt * addRefStmt; - if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", - -1, &addRefStmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); - foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); foreach (PathSet::iterator, j, info.references) { - if (sqlite3_reset(addRefStmt) != SQLITE_OK) + if (sqlite3_reset(stmtAddReference) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); - if (sqlite3_bind_int(addRefStmt, 1, pathToId[*i]) != SQLITE_OK) + if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); if (pathToId.find(*j) == pathToId.end()) throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); - if (sqlite3_bind_int(addRefStmt, 2, pathToId[*j]) != SQLITE_OK) + if (sqlite3_bind_int(stmtAddReference, 2, pathToId[*j]) != SQLITE_OK) throw SQLiteError(db, "binding argument 2"); - if (sqlite3_step(addRefStmt) != SQLITE_DONE) + if (sqlite3_step(stmtAddReference) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); } @@ -1287,12 +1321,6 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `commit' command"); - - if (sqlite3_finalize(addRefStmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); - - throw Error("foo"); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f29565680..3637a5d07 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -6,7 +6,9 @@ #include "store-api.hh" #include "util.hh" -#include + +class sqlite3; +class sqlite3_stmt; namespace nix { @@ -14,8 +16,9 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12*/ -const int nixSchemaVersion = 5; + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.14. Version 6 is + Nix 0.15. */ +const int nixSchemaVersion = 6; extern string drvsLogDir; @@ -43,6 +46,28 @@ struct RunningSubstituter }; +/* Wrapper object to close the SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + + +/* Wrapper object to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db; + sqlite3_stmt * stmt; + SQLiteStmt() { stmt = 0; } + void create(sqlite3 * db, const string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } +}; + + class LocalStore : public StoreAPI { private: @@ -163,12 +188,19 @@ private: /* Whether to do an fsync() after writing Nix metadata. */ bool doFsync; - sqlite3 * db; + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtAddReference; int getSchema(); void initSchema(); + void prepareStatements(); + void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false); From cfb09e0fadf7db240f4f8c35c35c13b192456b89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 14:40:07 +0000 Subject: [PATCH 006/128] * Automatically abort transactions if they go out of scope without committing. --- src/libstore/local-store.cc | 43 +++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 870644c88..ac6d768a0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -65,6 +65,37 @@ SQLiteStmt::~SQLiteStmt() } +struct SQLiteTxn +{ + bool active; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db) : active(false) { + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "starting transaction"); + active = true; + } + + void commit() + { + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "committing transaction"); + active = false; + } + + ~SQLiteTxn() + { + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } + } +}; + + void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -130,7 +161,8 @@ LocalStore::LocalStore() throw SQLiteError(db, "setting timeout"); /* Check the current database schema and if necessary do an - upgrade. */ + upgrade. !!! Race condition: several processes could start + the upgrade at the same time. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") @@ -1264,9 +1296,8 @@ void LocalStore::upgradeStore6() PathSet validPaths = queryValidPaths(); - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "running `begin' command"); - + SQLiteTxn txn(db); + std::map pathToId; foreach (PathSet::iterator, i, validPaths) { @@ -1319,8 +1350,8 @@ void LocalStore::upgradeStore6() std::cerr << "\n"; - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "running `commit' command"); + txn.commit(); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); From 885e22b16e3a6ea2a94014d33de31d756fa32eda Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 15:11:08 +0000 Subject: [PATCH 007/128] * Implement isValidPath(). --- src/libstore/local-store.cc | 244 ++++++++++++++---------------------- src/libstore/local-store.hh | 15 +-- 2 files changed, 100 insertions(+), 159 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ac6d768a0..05133ba55 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -54,6 +54,14 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) } +void SQLiteStmt::reset() +{ + assert(stmt); + if (sqlite3_reset(stmt) != SQLITE_OK) + throw SQLiteError(db, "resetting statement"); +} + + SQLiteStmt::~SQLiteStmt() { try { @@ -188,14 +196,11 @@ LocalStore::LocalStore() LocalStore::~LocalStore() { try { - flushDelayedUpdates(); - foreach (RunningSubstituters::iterator, i, runningSubstituters) { i->second.to.close(); i->second.from.close(); i->second.pid.wait(true); } - } catch (...) { ignoreException(); } @@ -231,6 +236,7 @@ void LocalStore::prepareStatements() "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, "insert into Refs (referrer, reference) values (?, ?);"); + stmtIsValidPath.create(db, "select 1 from ValidPaths where path = ?;"); } @@ -308,98 +314,6 @@ void canonicalisePathMetaData(const Path & path) } -static Path infoFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/info/%2%") % nixDBPath % baseName).str(); -} - - -static Path referrersFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/referrer/%2%") % nixDBPath % baseName).str(); -} - - -static Path failedFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/failed/%2%") % nixDBPath % baseName).str(); -} - - -static Path tmpFileForAtomicUpdate(const Path & path) -{ - return (format("%1%/.%2%.%3%") % dirOf(path) % getpid() % baseNameOf(path)).str(); -} - - -void LocalStore::appendReferrer(const Path & from, const Path & to, bool lock) -{ - Path referrersFile = referrersFileFor(from); - - PathLocks referrersLock; - if (lock) { - referrersLock.lockPaths(singleton(referrersFile)); - referrersLock.setDeletion(true); - } - - AutoCloseFD fd = open(referrersFile.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0666); - if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); - - string s = " " + to; - writeFull(fd, (const unsigned char *) s.c_str(), s.size()); - - if (doFsync) fdatasync(fd); -} - - -/* Atomically update the referrers file. If `purge' is true, the set - of referrers is set to `referrers'. Otherwise, the current set of - referrers is purged of invalid paths. */ -void LocalStore::rewriteReferrers(const Path & path, bool purge, PathSet referrers) -{ - Path referrersFile = referrersFileFor(path); - - PathLocks referrersLock(singleton(referrersFile)); - referrersLock.setDeletion(true); - - if (purge) - /* queryReferrers() purges invalid paths, so that's all we - need. */ - queryReferrers(path, referrers); - - Path tmpFile = tmpFileForAtomicUpdate(referrersFile); - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); - if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); - - string s; - foreach (PathSet::const_iterator, i, referrers) { - s += " "; s += *i; - } - - writeFull(fd, (const unsigned char *) s.c_str(), s.size()); - - if (doFsync) fdatasync(fd); - - fd.close(); /* for Windows; can't rename open file */ - - if (rename(tmpFile.c_str(), referrersFile.c_str()) == -1) - throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % referrersFile); -} - - -void LocalStore::flushDelayedUpdates() -{ - foreach (PathSet::iterator, i, delayedUpdates) { - rewriteReferrers(*i, true, PathSet()); - } - delayedUpdates.clear(); -} - - void LocalStore::registerValidPath(const Path & path, const Hash & hash, const PathSet & references, const Path & deriver) @@ -415,6 +329,7 @@ void LocalStore::registerValidPath(const Path & path, void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity) { +#if 0 Path infoFile = infoFileFor(info.path); ValidPathInfo oldInfo; @@ -467,22 +382,25 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi writeFile(tmpFile, s, doFsync); if (rename(tmpFile.c_str(), infoFile.c_str()) == -1) throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile); - - pathInfoCache[info.path] = info; +#endif } void LocalStore::registerFailedPath(const Path & path) { +#if 0 /* Write an empty file in the .../failed directory to denote the failure of the builder for `path'. */ writeFile(failedFileFor(path), ""); +#endif } bool LocalStore::hasPathFailed(const Path & path) { +#if 0 return pathExists(failedFileFor(path)); +#endif } @@ -502,6 +420,7 @@ Hash parseHashField(const Path & path, const string & s) ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) { +#if 0 ValidPathInfo res; res.path = path; @@ -510,9 +429,6 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) if (!isValidPath(path)) throw Error(format("path `%1%' is not valid") % path); - std::map::iterator lookup = pathInfoCache.find(path); - if (lookup != pathInfoCache.end()) return lookup->second; - /* Read the info file. */ Path infoFile = infoFileFor(path); if (!pathExists(infoFile)) @@ -550,31 +466,20 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) } } - return pathInfoCache[path] = res; + return res; +#endif } bool LocalStore::isValidPath(const Path & path) { - /* Files in the info directory starting with a `.' are temporary - files. */ - if (baseNameOf(path).at(0) == '.') return false; - - /* A path is valid if its info file exists and has a non-zero - size. (The non-zero size restriction is to be robust to - certain kinds of filesystem corruption, particularly with - ext4.) */ - Path infoFile = infoFileFor(path); - - struct stat st; - if (lstat(infoFile.c_str(), &st)) { - if (errno == ENOENT) return false; - throw SysError(format("getting status of `%1%'") % infoFile); - } - - if (st.st_size == 0) return false; - - return true; + stmtIsValidPath.reset(); + if (sqlite3_bind_text(stmtIsValidPath, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + int res = sqlite3_step(stmtIsValidPath); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throw SQLiteError(db, "querying path in database"); + return res == SQLITE_ROW; } @@ -598,6 +503,7 @@ void LocalStore::queryReferences(const Path & path, bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers) { +#if 0 bool allValid = true; if (!isValidPath(path)) @@ -623,6 +529,7 @@ bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers) if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false; return allValid; +#endif } @@ -788,6 +695,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { +#if 0 debug(format("invalidating path `%1%'") % path); ValidPathInfo info; @@ -805,28 +713,7 @@ void LocalStore::invalidatePath(const Path & path) Path p = referrersFileFor(path); if (pathExists(p) && unlink(p.c_str()) == -1) throw SysError(format("unlinking `%1%'") % p); - - /* Clear `path' from the info cache. */ - pathInfoCache.erase(path); - delayedUpdates.erase(path); - - /* Cause the referrer files for each path referenced by this one - to be updated. This has to happen after removing the info file - to preserve the invariant (see registerValidPath()). - - The referrer files are updated lazily in flushDelayedUpdates() - to prevent quadratic performance in the garbage collector - (i.e., when N referrers to some path X are deleted, we have to - rewrite the referrers file for X N times, causing O(N^2) I/O). - - What happens if we die before the referrer file can be updated? - That's not a problem, because stale (invalid) entries in the - referrer file are ignored by queryReferrers(). Thus a referrer - file is allowed to have stale entries; removing them is just an - optimisation. verifyStore() gets rid of them eventually. - */ - foreach (PathSet::iterator, i, info.references) - if (*i != path) delayedUpdates.insert(*i); +#endif } @@ -1130,6 +1017,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { +#if 0 bytesFreed = 0; assertStorePath(path); @@ -1149,11 +1037,13 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr } deletePathWrapped(path, bytesFreed, blocksFreed); +#endif } void LocalStore::verifyStore(bool checkContents) { +#if 0 /* Check whether all valid paths actually exist. */ printMsg(lvlInfo, "checking path existence"); @@ -1279,6 +1169,64 @@ void LocalStore::verifyStore(bool checkContents) if (update) rewriteReferrers(from, false, referrersNew); } +#endif +} + + +/* Functions for upgrading from the pre-SQLite database. */ + +static Path infoFileFor(const Path & path) +{ + string baseName = baseNameOf(path); + return (format("%1%/info/%2%") % nixDBPath % baseName).str(); +} + + +PathSet LocalStore::queryValidPathsOld() +{ + PathSet paths; + Strings entries = readDirectory(nixDBPath + "/info"); + foreach (Strings::iterator, i, entries) + if (i->at(0) != '.') paths.insert(nixStore + "/" + *i); + return paths; +} + + +ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) +{ + ValidPathInfo res; + res.path = path; + + /* Read the info file. */ + Path infoFile = infoFileFor(path); + if (!pathExists(infoFile)) + throw Error(format("path `%1%' is not valid") % path); + string info = readFile(infoFile); + + /* Parse it. */ + Strings lines = tokenizeString(info, "\n"); + + foreach (Strings::iterator, i, lines) { + string::size_type p = i->find(':'); + if (p == string::npos) + throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i); + string name(*i, 0, p); + string value(*i, p + 2); + if (name == "References") { + Strings refs = tokenizeString(value, " "); + res.references = PathSet(refs.begin(), refs.end()); + } else if (name == "Deriver") { + res.deriver = value; + } else if (name == "Hash") { + res.hash = parseHashField(path, value); + } else if (name == "Registered-At") { + int n = 0; + string2Int(value, n); + res.registrationTime = n; + } + } + + return res; } @@ -1294,17 +1242,16 @@ void LocalStore::upgradeStore6() initSchema(); - PathSet validPaths = queryValidPaths(); + PathSet validPaths = queryValidPathsOld(); SQLiteTxn txn(db); std::map pathToId; foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfo(*i, true); + ValidPathInfo info = queryPathInfoOld(*i); - if (sqlite3_reset(stmtRegisterValidPath) != SQLITE_OK) - throw SQLiteError(db, "resetting statement"); + stmtRegisterValidPath.reset(); if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); string h = "sha256:" + printHash(info.hash); @@ -1330,11 +1277,10 @@ void LocalStore::upgradeStore6() std::cerr << "|"; foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfo(*i, true); + ValidPathInfo info = queryPathInfoOld(*i); foreach (PathSet::iterator, j, info.references) { - if (sqlite3_reset(stmtAddReference) != SQLITE_OK) - throw SQLiteError(db, "resetting statement"); + stmtAddReference.reset(); if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); if (pathToId.find(*j) == pathToId.end()) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3637a5d07..fe4ed035d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -63,6 +63,7 @@ struct SQLiteStmt sqlite3_stmt * stmt; SQLiteStmt() { stmt = 0; } void create(sqlite3 * db, const string & s); + void reset(); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } }; @@ -178,13 +179,6 @@ private: /* Lock file used for upgrading. */ AutoCloseFD globalLock; - /* !!! The cache can grow very big. Maybe it should be pruned - every once in a while. */ - std::map pathInfoCache; - - /* Store paths for which the referrers file must be purged. */ - PathSet delayedUpdates; - /* Whether to do an fsync() after writing Nix metadata. */ bool doFsync; @@ -194,6 +188,7 @@ private: /* Some precompiled SQLite statements. */ SQLiteStmt stmtRegisterValidPath; SQLiteStmt stmtAddReference; + SQLiteStmt stmtIsValidPath; int getSchema(); @@ -209,13 +204,13 @@ private: void rewriteReferrers(const Path & path, bool purge, PathSet referrers); - void flushDelayedUpdates(); - bool queryReferrersInternal(const Path & path, PathSet & referrers); void invalidatePath(const Path & path); - + void upgradeStore6(); + PathSet queryValidPathsOld(); + ValidPathInfo queryPathInfoOld(const Path & path); struct GCState; From 77cb9e3fb19005a40724cb42773d2b11e2528858 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 15:52:57 +0000 Subject: [PATCH 008/128] * Implement queryPathInfo(). --- src/libstore/local-store.cc | 95 ++++++++++++++++++------------------- src/libstore/local-store.hh | 5 +- 2 files changed, 49 insertions(+), 51 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 05133ba55..76911da67 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -236,7 +236,10 @@ void LocalStore::prepareStatements() "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, "insert into Refs (referrer, reference) values (?, ?);"); - stmtIsValidPath.create(db, "select 1 from ValidPaths where path = ?;"); + stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); + stmtQueryReferences.create(db, + "select path from Refs join ValidPaths on reference = id where referrer = ?;"); } @@ -418,65 +421,59 @@ Hash parseHashField(const Path & path, const string & s) } -ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) +ValidPathInfo LocalStore::queryPathInfo(const Path & path) { -#if 0 ValidPathInfo res; res.path = path; assertStorePath(path); - if (!isValidPath(path)) - throw Error(format("path `%1%' is not valid") % path); + /* Get the path info. */ + stmtQueryPathInfo.reset(); + + if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + + int r = sqlite3_step(stmtQueryPathInfo); + if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); - /* Read the info file. */ - Path infoFile = infoFileFor(path); - if (!pathExists(infoFile)) - throw Error(format("path `%1%' is not valid") % path); - string info = readFile(infoFile); + unsigned int id = sqlite3_column_int(stmtQueryPathInfo, 0); - /* Parse it. */ - Strings lines = tokenizeString(info, "\n"); + const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); + assert(s); + res.hash = parseHashField(path, s); + + res.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); - foreach (Strings::iterator, i, lines) { - string::size_type p = i->find(':'); - if (p == string::npos) { - if (!ignoreErrors) - throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i); - continue; /* bad line */ - } - string name(*i, 0, p); - string value(*i, p + 2); - if (name == "References") { - Strings refs = tokenizeString(value, " "); - res.references = PathSet(refs.begin(), refs.end()); - } else if (name == "Deriver") { - res.deriver = value; - } else if (name == "Hash") { - try { - res.hash = parseHashField(path, value); - } catch (Error & e) { - if (!ignoreErrors) throw; - printMsg(lvlError, format("cannot parse hash field in `%1%': %2%") % infoFile % e.msg()); - } - } else if (name == "Registered-At") { - int n = 0; - string2Int(value, n); - res.registrationTime = n; - } + s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + if (s) res.deriver = s; + + /* Get the references. */ + stmtQueryReferences.reset(); + + if (sqlite3_bind_int(stmtQueryReferences, 1, id) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + + while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { + s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); + assert(s); + res.references.insert(s); } + if (r != SQLITE_DONE) + throw Error(format("error getting references of `%1%'") % path); + return res; -#endif } bool LocalStore::isValidPath(const Path & path) { - stmtIsValidPath.reset(); - if (sqlite3_bind_text(stmtIsValidPath, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + stmtQueryPathInfo.reset(); + if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); - int res = sqlite3_step(stmtIsValidPath); + int res = sqlite3_step(stmtQueryPathInfo); if (res != SQLITE_DONE && res != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); return res == SQLITE_ROW; @@ -1253,18 +1250,18 @@ void LocalStore::upgradeStore6() stmtRegisterValidPath.reset(); if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument 1"); + throw SQLiteError(db, "binding argument"); string h = "sha256:" + printHash(info.hash); if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument 2"); + throw SQLiteError(db, "binding argument"); if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) - throw SQLiteError(db, "binding argument 3"); + throw SQLiteError(db, "binding argument"); if (info.deriver != "") { if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument 4"); + throw SQLiteError(db, "binding argument"); } else { if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) - throw SQLiteError(db, "binding argument 4"); + throw SQLiteError(db, "binding argument"); } if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); @@ -1282,11 +1279,11 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, j, info.references) { stmtAddReference.reset(); if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) - throw SQLiteError(db, "binding argument 1"); + throw SQLiteError(db, "binding argument"); if (pathToId.find(*j) == pathToId.end()) throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); if (sqlite3_bind_int(stmtAddReference, 2, pathToId[*j]) != SQLITE_OK) - throw SQLiteError(db, "binding argument 2"); + throw SQLiteError(db, "binding argument"); if (sqlite3_step(stmtAddReference) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fe4ed035d..af1cd26a3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -188,7 +188,8 @@ private: /* Some precompiled SQLite statements. */ SQLiteStmt stmtRegisterValidPath; SQLiteStmt stmtAddReference; - SQLiteStmt stmtIsValidPath; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; int getSchema(); @@ -198,7 +199,7 @@ private: void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); - ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false); + ValidPathInfo queryPathInfo(const Path & path); void appendReferrer(const Path & from, const Path & to, bool lock); From 836e5b6f5741c845da08490a6fd8ad9a3f37726e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 16:21:59 +0000 Subject: [PATCH 009/128] * Implemented queryReferrers(). --- src/libstore/local-store.cc | 54 +++++++++++++++---------------------- src/libstore/local-store.hh | 3 +-- 2 files changed, 22 insertions(+), 35 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 76911da67..f1deb2eae 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -167,6 +167,9 @@ LocalStore::LocalStore() if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) throw SQLiteError(db, "setting timeout"); + + /* !!! check whether sqlite has been built with foreign key + support */ /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start @@ -240,6 +243,8 @@ void LocalStore::prepareStatements() "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); + stmtQueryReferrers.create(db, + "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); } @@ -498,41 +503,24 @@ void LocalStore::queryReferences(const Path & path, } -bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers) -{ -#if 0 - bool allValid = true; - - if (!isValidPath(path)) - throw Error(format("path `%1%' is not valid") % path); - - /* No locking is necessary here: updates are only done by - appending or by atomically replacing the file. When appending, - there is a possibility that we see a partial entry, but it will - just be filtered out below (the partially written path will not - be valid, so it will be ignored). */ - - Path referrersFile = referrersFileFor(path); - if (!pathExists(referrersFile)) return true; - - AutoCloseFD fd = open(referrersFile.c_str(), O_RDONLY); - if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); - - Paths refs = tokenizeString(readFile(fd), " "); - - foreach (Paths::iterator, i, refs) - /* Referrers can be invalid (see registerValidPath() for the - invariant), so we only return one if it is valid. */ - if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false; - - return allValid; -#endif -} - - void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { - queryReferrersInternal(path, referrers); + assertStorePath(path); + + stmtQueryReferrers.reset(); + + if (sqlite3_bind_text(stmtQueryReferrers, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + + int r; + while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); + assert(s); + referrers.insert(s); + } + + if (r != SQLITE_DONE) + throw Error(format("error getting references of `%1%'") % path); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index af1cd26a3..9eae443d1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -190,6 +190,7 @@ private: SQLiteStmt stmtAddReference; SQLiteStmt stmtQueryPathInfo; SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; int getSchema(); @@ -205,8 +206,6 @@ private: void rewriteReferrers(const Path & path, bool purge, PathSet referrers); - bool queryReferrersInternal(const Path & path, PathSet & referrers); - void invalidatePath(const Path & path); void upgradeStore6(); From 268f9aaf286294c05f34be338ac4ec75e20d1442 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 16:51:27 +0000 Subject: [PATCH 010/128] * Implemented queryValidPaths() and verifyStore(). --- src/libstore/local-store.cc | 151 ++++++++---------------------------- tests/init.sh | 3 +- 2 files changed, 35 insertions(+), 119 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f1deb2eae..7d6fc5e33 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,6 +56,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) void SQLiteStmt::reset() { + checkInterrupt(); assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); @@ -337,6 +338,7 @@ void LocalStore::registerValidPath(const Path & path, void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity) { + abort(); #if 0 Path infoFile = infoFileFor(info.path); @@ -396,19 +398,13 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi void LocalStore::registerFailedPath(const Path & path) { -#if 0 - /* Write an empty file in the .../failed directory to denote the - failure of the builder for `path'. */ - writeFile(failedFileFor(path), ""); -#endif + abort(); } bool LocalStore::hasPathFailed(const Path & path) { -#if 0 - return pathExists(failedFileFor(path)); -#endif + abort(); } @@ -467,7 +463,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } if (r != SQLITE_DONE) - throw Error(format("error getting references of `%1%'") % path); + throw SQLiteError(db, format("error getting references of `%1%'") % path); return res; } @@ -487,11 +483,22 @@ bool LocalStore::isValidPath(const Path & path) PathSet LocalStore::queryValidPaths() { - PathSet paths; - Strings entries = readDirectory(nixDBPath + "/info"); - foreach (Strings::iterator, i, entries) - if (i->at(0) != '.') paths.insert(nixStore + "/" + *i); - return paths; + SQLiteStmt stmt; + stmt.create(db, "select path from ValidPaths"); + + PathSet res; + + int r; + while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmt, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, "error getting valid paths"); + + return res; } @@ -520,7 +527,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } if (r != SQLITE_DONE) - throw Error(format("error getting references of `%1%'") % path); + throw SQLiteError(db, format("error getting references of `%1%'") % path); } @@ -680,6 +687,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { + abort(); #if 0 debug(format("invalidating path `%1%'") % path); @@ -1002,6 +1010,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { + abort(); #if 0 bytesFreed = 0; @@ -1028,7 +1037,6 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr void LocalStore::verifyStore(bool checkContents) { -#if 0 /* Check whether all valid paths actually exist. */ printMsg(lvlInfo, "checking path existence"); @@ -1036,68 +1044,26 @@ void LocalStore::verifyStore(bool checkContents) foreach (PathSet::iterator, i, validPaths2) { checkInterrupt(); + /* !!! invalidatePath() will probably fail due to the foreign + key constraints on the Ref table. */ if (!isStorePath(*i)) { printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i); invalidatePath(*i); } else if (!pathExists(*i)) { printMsg(lvlError, format("path `%1%' disappeared") % *i); invalidatePath(*i); - } else { - Path infoFile = infoFileFor(*i); - struct stat st; - if (lstat(infoFile.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % infoFile); - if (st.st_size == 0) { - printMsg(lvlError, format("removing corrupt info file `%1%'") % infoFile); - if (unlink(infoFile.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % infoFile); - } - else validPaths.insert(*i); - } + } else validPaths.insert(*i); } + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + printMsg(lvlInfo, "checking hashes"); - /* Check the store path meta-information. */ - printMsg(lvlInfo, "checking path meta-information"); + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfo(*i); - std::map referrersCache; - - foreach (PathSet::iterator, i, validPaths) { - bool update = false; - ValidPathInfo info = queryPathInfo(*i, true); - - /* Check the references: each reference should be valid, and - it should have a matching referrer. */ - foreach (PathSet::iterator, j, info.references) { - if (validPaths.find(*j) == validPaths.end()) { - printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") - % *i % *j); - /* nothing we can do about it... */ - } else { - if (referrersCache.find(*j) == referrersCache.end()) - queryReferrers(*j, referrersCache[*j]); - if (referrersCache[*j].find(*i) == referrersCache[*j].end()) { - printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'") - % *j % *i); - appendReferrer(*j, *i, true); - } - } - } - - /* Check the deriver. (Note that the deriver doesn't have to - be a valid path.) */ - if (!info.deriver.empty() && !isStorePath(info.deriver)) { - info.deriver = ""; - update = true; - } - - /* Check the content hash (optionally - slow). */ - if (info.hash.hashSize == 0) { - printMsg(lvlError, format("re-hashing `%1%'") % *i); - info.hash = hashPath(htSHA256, *i); - update = true; - } else if (checkContents) { - debug(format("checking contents of `%1%'") % *i); + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); Hash current = hashPath(info.hash.type, *i); if (current != info.hash) { printMsg(lvlError, format("path `%1%' was modified! " @@ -1105,56 +1071,7 @@ void LocalStore::verifyStore(bool checkContents) % *i % printHash(info.hash) % printHash(current)); } } - - if (update) registerValidPath(info); } - - referrersCache.clear(); - - - /* Check the referrers. */ - printMsg(lvlInfo, "checking referrers"); - - std::map referencesCache; - - Strings entries = readDirectory(nixDBPath + "/referrer"); - foreach (Strings::iterator, i, entries) { - Path from = nixStore + "/" + *i; - - if (validPaths.find(from) == validPaths.end()) { - /* !!! This removes lock files as well. Need to check - whether that's okay. */ - printMsg(lvlError, format("removing referrers file for invalid `%1%'") % from); - Path p = referrersFileFor(from); - if (unlink(p.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % p); - continue; - } - - PathSet referrers; - bool allValid = queryReferrersInternal(from, referrers); - bool update = false; - - if (!allValid) { - printMsg(lvlError, format("removing some stale referrers for `%1%'") % from); - update = true; - } - - /* Each referrer should have a matching reference. */ - PathSet referrersNew; - foreach (PathSet::iterator, j, referrers) { - if (referencesCache.find(*j) == referencesCache.end()) - queryReferences(*j, referencesCache[*j]); - if (referencesCache[*j].find(from) == referencesCache[*j].end()) { - printMsg(lvlError, format("removing unexpected referrer mapping from `%1%' to `%2%'") - % from % *j); - update = true; - } else referrersNew.insert(*j); - } - - if (update) rewriteReferrers(from, false, referrersNew); - } -#endif } diff --git a/tests/init.sh b/tests/init.sh index 0639a7066..691cb669b 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -96,7 +96,6 @@ mv $NIX_BIN_DIR/nix/download-using-manifests.pl $NIX_BIN_DIR/nix/substituters/do $nixstore --init # Did anything happen? -test -e "$NIX_DB_DIR"/info -test -e "$NIX_DB_DIR"/referrer +test -e "$NIX_DB_DIR"/db.sqlite echo 'Hello World' > ./dummy From 762cee72ccd860e72c7b639a1dd542ac0f298bb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Feb 2010 16:04:51 +0000 Subject: [PATCH 011/128] * Implement registerValidPath(). --- src/libstore/local-store.cc | 201 ++++++++++++++++-------------------- src/libstore/local-store.hh | 6 +- src/libstore/store-api.hh | 1 + 3 files changed, 96 insertions(+), 112 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7d6fc5e33..af240b61c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -74,6 +74,24 @@ SQLiteStmt::~SQLiteStmt() } +/* Helper class to ensure that prepared statements are reset when + leaving the scope that uses them. Unfinished prepared statements + prevent transactions from being aborted, and can cause locks to be + kept when they should be released. */ +struct SQLiteStmtUse +{ + SQLiteStmt & stmt; + SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) + { + stmt.reset(); + } + ~SQLiteStmtUse() + { + stmt.reset(); + } +}; + + struct SQLiteTxn { bool active; @@ -223,10 +241,10 @@ int LocalStore::getSchema() } -#include "schema.sql.hh" - void LocalStore::initSchema() { +#include "schema.sql.hh" + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "initialising database schema"); @@ -237,9 +255,9 @@ void LocalStore::initSchema() void LocalStore::prepareStatements() { stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + "insert or replace into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, - "insert into Refs (referrer, reference) values (?, ?);"); + "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); stmtQueryReferences.create(db, @@ -336,75 +354,69 @@ void LocalStore::registerValidPath(const Path & path, } -void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity) +unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) { - abort(); -#if 0 - Path infoFile = infoFileFor(info.path); - - ValidPathInfo oldInfo; - if (pathExists(infoFile)) oldInfo = queryPathInfo(info.path); - - /* Note that it's possible for infoFile to already exist. */ - - /* Acquire a lock on each referrer file. This prevents those - paths from being invalidated. (It would be a violation of the - store invariants if we registered info.path as valid while some - of its references are invalid.) NB: there can be no deadlock - here since we're acquiring the locks in sorted order. */ - PathSet lockNames; - foreach (PathSet::const_iterator, i, info.references) - if (*i != info.path) lockNames.insert(referrersFileFor(*i)); - PathLocks referrerLocks(lockNames); - referrerLocks.setDeletion(true); - - string refs; - foreach (PathSet::const_iterator, i, info.references) { - if (!refs.empty()) refs += " "; - refs += *i; - - if (!ignoreValidity && *i != info.path && !isValidPath(*i)) - throw Error(format("cannot register `%1%' as valid, because its reference `%2%' isn't valid") - % info.path % *i); - - /* Update the referrer mapping for *i. This must be done - before the info file is written to maintain the invariant - that if `path' is a valid path, then all its references - have referrer mappings back to `path'. A " " is prefixed - to separate it from the previous entry. It's not suffixed - to deal with interrupted partial writes to this file. */ - if (oldInfo.references.find(*i) == oldInfo.references.end()) - appendReferrer(*i, info.path, false); + SQLiteStmtUse use(stmtRegisterValidPath); + if (sqlite3_bind_text(stmtRegisterValidPath, 1, info.path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + string h = "sha256:" + printHash(info.hash); + if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (info.deriver != "") { + if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + } else { + if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); } + if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) + throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); + return sqlite3_last_insert_rowid(db); +} + +void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) +{ + SQLiteStmtUse use(stmtAddReference); + if (sqlite3_bind_int(stmtAddReference, 1, referrer) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (sqlite3_bind_int(stmtAddReference, 2, reference) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (sqlite3_step(stmtAddReference) != SQLITE_DONE) + throw SQLiteError(db, "adding reference to database"); +} + + +void LocalStore::registerValidPath(const ValidPathInfo & info) +{ assert(info.hash.type == htSHA256); + ValidPathInfo info2(info); + if (info2.registrationTime == 0) info2.registrationTime = time(0); + + SQLiteTxn txn(db); + + unsigned long long id = addValidPath(info2); - string s = (format( - "Hash: sha256:%1%\n" - "References: %2%\n" - "Deriver: %3%\n" - "Registered-At: %4%\n") - % printHash(info.hash) % refs % info.deriver % - (oldInfo.registrationTime ? oldInfo.registrationTime : time(0))).str(); - - /* Atomically rewrite the info file. */ - Path tmpFile = tmpFileForAtomicUpdate(infoFile); - writeFile(tmpFile, s, doFsync); - if (rename(tmpFile.c_str(), infoFile.c_str()) == -1) - throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile); -#endif + foreach (PathSet::const_iterator, i, info2.references) { + ValidPathInfo ref = queryPathInfo(*i); + addReference(id, ref.id); + } + + txn.commit(); } void LocalStore::registerFailedPath(const Path & path) { - abort(); + throw Error("not implemented"); } bool LocalStore::hasPathFailed(const Path & path) { - abort(); + throw Error("not implemented"); } @@ -424,13 +436,13 @@ Hash parseHashField(const Path & path, const string & s) ValidPathInfo LocalStore::queryPathInfo(const Path & path) { - ValidPathInfo res; - res.path = path; + ValidPathInfo info; + info.path = path; assertStorePath(path); /* Get the path info. */ - stmtQueryPathInfo.reset(); + SQLiteStmtUse use1(stmtQueryPathInfo); if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); @@ -439,39 +451,39 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); - unsigned int id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.id = sqlite3_column_int(stmtQueryPathInfo, 0); const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); assert(s); - res.hash = parseHashField(path, s); + info.hash = parseHashField(path, s); - res.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); - if (s) res.deriver = s; + if (s) info.deriver = s; /* Get the references. */ - stmtQueryReferences.reset(); + SQLiteStmtUse use2(stmtQueryReferences); - if (sqlite3_bind_int(stmtQueryReferences, 1, id) != SQLITE_OK) + if (sqlite3_bind_int(stmtQueryReferences, 1, info.id) != SQLITE_OK) throw SQLiteError(db, "binding argument"); while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); assert(s); - res.references.insert(s); + info.references.insert(s); } if (r != SQLITE_DONE) throw SQLiteError(db, format("error getting references of `%1%'") % path); - return res; + return info; } bool LocalStore::isValidPath(const Path & path) { - stmtQueryPathInfo.reset(); + SQLiteStmtUse use(stmtQueryPathInfo); if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); int res = sqlite3_step(stmtQueryPathInfo); @@ -514,7 +526,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - stmtQueryReferrers.reset(); + SQLiteStmtUse use(stmtQueryReferrers); if (sqlite3_bind_text(stmtQueryReferrers, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); @@ -687,7 +699,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { - abort(); + throw Error("not implemented"); #if 0 debug(format("invalidating path `%1%'") % path); @@ -1010,7 +1022,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { - abort(); + throw Error("not implemented"); + #if 0 bytesFreed = 0; @@ -1077,13 +1090,6 @@ void LocalStore::verifyStore(bool checkContents) /* Functions for upgrading from the pre-SQLite database. */ -static Path infoFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/info/%2%") % nixDBPath % baseName).str(); -} - - PathSet LocalStore::queryValidPathsOld() { PathSet paths; @@ -1100,7 +1106,8 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) res.path = path; /* Read the info file. */ - Path infoFile = infoFileFor(path); + string baseName = baseNameOf(path); + Path infoFile = (format("%1%/info/%2%") % nixDBPath % baseName).str(); if (!pathExists(infoFile)) throw Error(format("path `%1%' is not valid") % path); string info = readFile(infoFile); @@ -1152,27 +1159,7 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfoOld(*i); - - stmtRegisterValidPath.reset(); - if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - string h = "sha256:" + printHash(info.hash); - if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (info.deriver != "") { - if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } else { - if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throw SQLiteError(db, "registering valid path in database"); - - pathToId[*i] = sqlite3_last_insert_rowid(db); - + pathToId[*i] = addValidPath(info); std::cerr << "."; } @@ -1180,19 +1167,11 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfoOld(*i); - foreach (PathSet::iterator, j, info.references) { - stmtAddReference.reset(); - if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); if (pathToId.find(*j) == pathToId.end()) throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); - if (sqlite3_bind_int(stmtAddReference, 2, pathToId[*j]) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throw SQLiteError(db, "adding reference to database"); + addReference(pathToId[*i], pathToId[*j]); } - std::cerr << "."; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9eae443d1..cfe26039e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,7 +198,11 @@ private: void prepareStatements(); - void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); + unsigned long long addValidPath(const ValidPathInfo & info); + + void addReference(unsigned long long referrer, unsigned long long reference); + + void registerValidPath(const ValidPathInfo & info); ValidPathInfo queryPathInfo(const Path & path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8506d47e3..23a3ab25e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -333,6 +333,7 @@ struct ValidPathInfo Hash hash; PathSet references; time_t registrationTime; + unsigned long long id; // internal use only ValidPathInfo() : registrationTime(0) { } }; From 9c9a88e9e25bdc4456368a40691e61acf5d3b330 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Feb 2010 16:43:25 +0000 Subject: [PATCH 012/128] * Implement more stuff. --- src/libstore/local-store.cc | 97 +++++++++++++++++-------------------- src/libstore/local-store.hh | 5 ++ 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index af240b61c..cd25ded72 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -60,6 +60,7 @@ void SQLiteStmt::reset() assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); + curArg = 1; } @@ -74,6 +75,27 @@ SQLiteStmt::~SQLiteStmt() } +void SQLiteStmt::bind(const string & value) +{ + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind(int value) +{ + if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + /* Helper class to ensure that prepared statements are reset when leaving the scope that uses them. Unfinished prepared statements prevent transactions from being aborted, and can cause locks to be @@ -264,6 +286,8 @@ void LocalStore::prepareStatements() "select path from Refs join ValidPaths on reference = id where referrer = ?;"); stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); + stmtInvalidatePath.create(db, + "delete from ValidPaths where path = ?;"); } @@ -357,20 +381,13 @@ void LocalStore::registerValidPath(const Path & path, unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) { SQLiteStmtUse use(stmtRegisterValidPath); - if (sqlite3_bind_text(stmtRegisterValidPath, 1, info.path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - string h = "sha256:" + printHash(info.hash); - if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (info.deriver != "") { - if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } else { - if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } + stmtRegisterValidPath.bind(info.path); + stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); + stmtRegisterValidPath.bind(info.registrationTime); + if (info.deriver != "") + stmtRegisterValidPath.bind(info.deriver); + else + stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); return sqlite3_last_insert_rowid(db); @@ -380,10 +397,8 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) { SQLiteStmtUse use(stmtAddReference); - if (sqlite3_bind_int(stmtAddReference, 1, referrer) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_bind_int(stmtAddReference, 2, reference) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtAddReference.bind(referrer); + stmtAddReference.bind(reference); if (sqlite3_step(stmtAddReference) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); } @@ -443,9 +458,8 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) /* Get the path info. */ SQLiteStmtUse use1(stmtQueryPathInfo); - - if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + + stmtQueryPathInfo.bind(path); int r = sqlite3_step(stmtQueryPathInfo); if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); @@ -465,8 +479,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) /* Get the references. */ SQLiteStmtUse use2(stmtQueryReferences); - if (sqlite3_bind_int(stmtQueryReferences, 1, info.id) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtQueryReferences.bind(info.id); while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); @@ -484,8 +497,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) bool LocalStore::isValidPath(const Path & path) { SQLiteStmtUse use(stmtQueryPathInfo); - if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtQueryPathInfo.bind(path); int res = sqlite3_step(stmtQueryPathInfo); if (res != SQLITE_DONE && res != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); @@ -528,8 +540,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) SQLiteStmtUse use(stmtQueryReferrers); - if (sqlite3_bind_text(stmtQueryReferrers, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtQueryReferrers.bind(path); int r; while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { @@ -699,26 +710,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { - throw Error("not implemented"); -#if 0 debug(format("invalidating path `%1%'") % path); + + SQLiteStmtUse use(stmtInvalidatePath); - ValidPathInfo info; + stmtInvalidatePath.bind(path); - if (pathExists(infoFileFor(path))) { - info = queryPathInfo(path); + if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) + throw SQLiteError(db, format("invalidating path `%1%' in database") % path); - /* Remove the info file. */ - Path p = infoFileFor(path); - if (unlink(p.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % p); - } - - /* Remove the referrers file for `path'. */ - Path p = referrersFileFor(path); - if (pathExists(p) && unlink(p.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % p); -#endif + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ } @@ -1022,19 +1024,11 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { - throw Error("not implemented"); - -#if 0 bytesFreed = 0; assertStorePath(path); if (isValidPath(path)) { - /* Acquire a lock on the referrers file to prevent new - referrers to this path from appearing while we're deleting - it. */ - PathLocks referrersLock(singleton(referrersFileFor(path))); - referrersLock.setDeletion(true); PathSet referrers; queryReferrers(path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) @@ -1044,7 +1038,6 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr } deletePathWrapped(path, bytesFreed, blocksFreed); -#endif } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index cfe26039e..34aa55fc8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -61,11 +61,15 @@ struct SQLiteStmt { sqlite3 * db; sqlite3_stmt * stmt; + unsigned int curArg; SQLiteStmt() { stmt = 0; } void create(sqlite3 * db, const string & s); void reset(); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } + void bind(const string & value); + void bind(int value); + void bind(); }; @@ -191,6 +195,7 @@ private: SQLiteStmt stmtQueryPathInfo; SQLiteStmt stmtQueryReferences; SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; int getSchema(); From 1930570ad96e47de9e8557a7734c7bfd9f36f942 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Feb 2010 17:15:22 +0000 Subject: [PATCH 013/128] * Foreign key support in SQLite is not a persistent setting, so enable it at startup. * Implement negative caching. Now `make check' passes. --- src/libstore/local-store.cc | 21 +++++++++++++++++++-- src/libstore/local-store.hh | 2 ++ src/libstore/schema.sql | 4 +--- tests/referrers.sh | 27 +++------------------------ 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cd25ded72..4c1bbb708 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -209,6 +209,9 @@ LocalStore::LocalStore() if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) throw SQLiteError(db, "setting timeout"); + if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "enabling foreign keys"); + /* !!! check whether sqlite has been built with foreign key support */ @@ -288,6 +291,10 @@ void LocalStore::prepareStatements() "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); + stmtRegisterFailedPath.create(db, + "insert into FailedPaths (path, time) values (?, ?);"); + stmtHasPathFailed.create(db, + "select time from FailedPaths where path = ?;"); } @@ -425,13 +432,23 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerFailedPath(const Path & path) { - throw Error("not implemented"); + if (hasPathFailed(path)) return; + SQLiteStmtUse use(stmtRegisterFailedPath); + stmtRegisterFailedPath.bind(path); + stmtRegisterFailedPath.bind(time(0)); + if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) + throw SQLiteError(db, format("registering failed path `%1%'") % path); } bool LocalStore::hasPathFailed(const Path & path) { - throw Error("not implemented"); + SQLiteStmtUse use(stmtHasPathFailed); + stmtHasPathFailed.bind(path); + int res = sqlite3_step(stmtHasPathFailed); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throw SQLiteError(db, "querying whether path failed"); + return res == SQLITE_ROW; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 34aa55fc8..e1859e68e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -196,6 +196,8 @@ private: SQLiteStmt stmtQueryReferences; SQLiteStmt stmtQueryReferrers; SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtRegisterFailedPath; + SQLiteStmt stmtHasPathFailed; int getSchema(); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index dc53f452c..1e707ce1f 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -1,5 +1,3 @@ -pragma foreign_keys = on; - create table if not exists ValidPaths ( id integer primary key autoincrement not null, path text unique not null, @@ -19,7 +17,7 @@ create table if not exists Refs ( create index if not exists IndexReferrer on Refs(referrer); create index if not exists IndexReference on Refs(reference); -create table if not exists FailedDerivations ( +create table if not exists FailedPaths ( path text primary key not null, time integer not null ); diff --git a/tests/referrers.sh b/tests/referrers.sh index e3f8e07bc..752f05c3d 100644 --- a/tests/referrers.sh +++ b/tests/referrers.sh @@ -3,6 +3,8 @@ source common.sh # This takes way to long on Cygwin (because process creation is so slow...). if test "$system" = i686-cygwin; then exit 0; fi +clearStore + max=2500 reference=$NIX_STORE_DIR/abcdef @@ -25,34 +27,11 @@ echo "registering..." time $nixstore --register-validity < $TEST_ROOT/reg_info -oldTime=$(cat test-tmp/db/info/1 | grep Registered-At) - -echo "sleeping..." - -sleep 2 - -echo "reregistering..." - -time $nixstore --register-validity --reregister < $TEST_ROOT/reg_info - -newTime=$(cat test-tmp/db/info/1 | grep Registered-At) - -if test "$newTime" != "$oldTime"; then - echo "reregistration changed original registration time" - exit 1 -fi - -if test "$(cat test-tmp/db/referrer/1 | wc -w)" -ne 1; then - echo "reregistration duplicated referrers" - exit 1 -fi - echo "collecting garbage..." ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref time $nixstore --gc -if test "$(cat test-tmp/db/referrer/abcdef | wc -w)" -ne 0; then +if test "$(sqlite3 ./test-tmp/db/db.sqlite 'select count(*) from Refs')" -ne 0; then echo "referrers not cleaned up" exit 1 fi - From 299ff64812ce166d230f1b630f794be226c7a178 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 11:15:50 +0000 Subject: [PATCH 014/128] * Put the derivation outputs in the database. This is useful for the garbage collector. --- src/libstore/local-store.cc | 26 ++++++++++++++++++++++++-- src/libstore/local-store.hh | 1 + src/libstore/schema.sql | 10 ++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4c1bbb708..2f12256db 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -3,9 +3,9 @@ #include "globals.hh" #include "archive.hh" #include "pathlocks.hh" -#include "aterm.hh" #include "derivations-ast.hh" #include "worker-protocol.hh" +#include "derivations.hh" #include #include @@ -295,6 +295,8 @@ void LocalStore::prepareStatements() "insert into FailedPaths (path, time) values (?, ?);"); stmtHasPathFailed.create(db, "select time from FailedPaths where path = ?;"); + stmtAddDerivationOutput.create(db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); } @@ -397,7 +399,27 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); - return sqlite3_last_insert_rowid(db); + unsigned long long id = sqlite3_last_insert_rowid(db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + ATerm t = ATreadFromNamedFile(info.path.c_str()); + if (!t) throw Error(format("cannot read derivation `%1%'") % info.path); + Derivation drv = parseDerivation(t); + foreach (DerivationOutputs::iterator, i, drv.outputs) { + SQLiteStmtUse use(stmtAddDerivationOutput); + stmtAddDerivationOutput.bind(id); + stmtAddDerivationOutput.bind(i->first); + stmtAddDerivationOutput.bind(i->second.path); + if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) + throw SQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + } + } + + return id; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e1859e68e..f10ba18d5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,6 +198,7 @@ private: SQLiteStmt stmtInvalidatePath; SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; + SQLiteStmt stmtAddDerivationOutput; int getSchema(); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 1e707ce1f..682ce5ed7 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -17,6 +17,16 @@ create table if not exists Refs ( create index if not exists IndexReferrer on Refs(referrer); create index if not exists IndexReference on Refs(reference); +create table if not exists DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); + +create index if not exists IndexDerivationOutputs on DerivationOutputs(path); + create table if not exists FailedPaths ( path text primary key not null, time integer not null From 103cfee056cbc8f002929fd5958e305c1a75fe45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 11:44:17 +0000 Subject: [PATCH 015/128] * Revert r19650 (implement gc-keep-outputs by looking for derivations with the same name as the output) and instead use the DerivationOutputs table in the database, which is the correct way to to do things. --- src/libstore/gc.cc | 56 ++----------------------------------- src/libstore/local-store.cc | 24 ++++++++++++++++ src/libstore/local-store.hh | 9 ++++-- src/libstore/store-api.cc | 13 --------- src/libstore/store-api.hh | 6 ---- 5 files changed, 34 insertions(+), 74 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f58f691c9..659c636e3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -416,12 +416,7 @@ struct LocalStore::GCState PathSet busy; bool gcKeepOutputs; bool gcKeepDerivations; - - bool drvsIndexed; - typedef std::multimap DrvsByName; - DrvsByName drvsByName; // derivation paths hashed by name attribute - - GCState(GCResults & results_) : results(results_), drvsIndexed(false) + GCState(GCResults & results_) : results(results_) { } }; @@ -441,42 +436,6 @@ bool LocalStore::isActiveTempFile(const GCState & state, && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); } - -/* Return all the derivations in the Nix store that have `path' as an - output. This function assumes that derivations have the same name - as their outputs. */ -PathSet LocalStore::findDerivers(GCState & state, const Path & path) -{ - PathSet derivers; - - Path deriver = queryDeriver(path); - if (deriver != "") derivers.insert(deriver); - - if (!state.drvsIndexed) { - Paths entries = readDirectory(nixStore); - foreach (Paths::iterator, i, entries) - if (isDerivation(*i)) - state.drvsByName.insert(std::pair( - getNameOfStorePath(*i), nixStore + "/" + *i)); - state.drvsIndexed = true; - } - - string name = getNameOfStorePath(path); - - // Urgh, I should have used Haskell... - std::pair range = - state.drvsByName.equal_range(name); - - for (GCState::DrvsByName::iterator i = range.first; i != range.second; ++i) - if (isValidPath(i->second)) { - Derivation drv = derivationFromPath(i->second); - foreach (DerivationOutputs::iterator, j, drv.outputs) - if (j->second.path == path) derivers.insert(i->second); - } - - return derivers; -} - bool LocalStore::tryToDelete(GCState & state, const Path & path) { @@ -522,18 +481,9 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) if (!pathExists(path)) return true; /* If gc-keep-outputs is set, then don't delete this path if - its deriver is not garbage. !!! Nix does not reliably - store derivers, so we have to look at all derivations to - determine which of them derive `path'. Since this makes - the garbage collector very slow to start on large Nix - stores, here we just look for all derivations that have the - same name as `path' (where the name is the part of the - filename after the hash, i.e. the `name' attribute of the - derivation). This is somewhat hacky: currently, the - deriver of a path always has the same name as the output, - but this might change in the future. */ + there are derivers of this path that are not garbage. */ if (state.gcKeepOutputs) { - PathSet derivers = findDerivers(state, path); + PathSet derivers = queryValidDerivers(path); foreach (PathSet::iterator, deriver, derivers) { /* Break an infinite recursion if gc-keep-derivations and gc-keep-outputs are both set by tentatively diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2f12256db..0590b294b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -297,6 +297,8 @@ void LocalStore::prepareStatements() "select time from FailedPaths where path = ?;"); stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); + stmtQueryValidDerivers.create(db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); } @@ -599,6 +601,28 @@ Path LocalStore::queryDeriver(const Path & path) } +PathSet LocalStore::queryValidDerivers(const Path & path) +{ + assertStorePath(path); + + SQLiteStmtUse use(stmtQueryValidDerivers); + stmtQueryValidDerivers.bind(path); + + PathSet derivers; + int r; + while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); + assert(s); + derivers.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, format("error getting valid derivers of `%1%'") % path); + + return derivers; +} + + void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) { if (run.pid != -1) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f10ba18d5..6bd47b305 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -103,6 +103,12 @@ public: void queryReferrers(const Path & path, PathSet & referrers); Path queryDeriver(const Path & path); + + /* Return all currently valid derivations that have `path' as an + output. (Note that the result of `queryDeriver()' is the + derivation that was actually used to produce `path', which may + not exist anymore.) */ + PathSet queryValidDerivers(const Path & path); PathSet querySubstitutablePaths(); @@ -199,6 +205,7 @@ private: SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; int getSchema(); @@ -228,8 +235,6 @@ private: bool tryToDelete(GCState & state, const Path & path); - PathSet findDerivers(GCState & state, const Path & path); - bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f0abe61ad..01dd51621 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,7 +1,6 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" -#include "derivations.hh" #include @@ -53,18 +52,6 @@ Path toStorePath(const Path & path) } -string getNameOfStorePath(const Path & path) -{ - Path::size_type slash = path.rfind('/'); - string p = slash == Path::npos ? path : string(path, slash + 1); - Path::size_type dash = p.find('-'); - assert(dash != Path::npos); - string p2 = string(p, dash + 1); - if (isDerivation(p2)) p2 = string(p2, 0, p2.size() - 4); - return p2; -} - - Path followLinksToStore(const Path & _path) { Path path = absPath(_path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 23a3ab25e..d85ae0c9a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -243,12 +243,6 @@ void checkStoreName(const string & name); Path toStorePath(const Path & path); -/* Get the "name" part of a store path, that is, the part after the - hash and the dash, and with any ".drv" suffix removed - (e.g. /nix/store/-foo-1.2.3.drv => foo-1.2.3). */ -string getNameOfStorePath(const Path & path); - - /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(const Path & path); From c4d388add4942f6f99a8df12f4e49149005047e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 12:44:36 +0000 Subject: [PATCH 016/128] * Get derivation outputs from the database instead of the .drv file, which requires more I/O. --- src/libstore/gc.cc | 8 ++++---- src/libstore/local-store.cc | 24 ++++++++++++++++++++++++ src/libstore/local-store.hh | 3 +++ src/libstore/misc.cc | 8 ++++---- src/libstore/remote-store.cc | 6 ++++++ src/libstore/remote-store.hh | 2 ++ src/libstore/store-api.hh | 3 +++ 7 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 659c636e3..cf073c5d9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -467,10 +467,10 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) then don't delete the derivation if any of the outputs are live. */ if (state.gcKeepDerivations && isDerivation(path)) { - Derivation drv = derivationFromPath(path); - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (!tryToDelete(state, i->second.path)) { - printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output is alive") % path); + PathSet outputs = queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (!tryToDelete(state, *i)) { + printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output `%2%' is alive") % path % *i); goto isLive; } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0590b294b..0823e785b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -299,6 +299,8 @@ void LocalStore::prepareStatements() "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); + stmtQueryDerivationOutputs.create(db, + "select id, path from DerivationOutputs where drv = ?;"); } @@ -623,6 +625,28 @@ PathSet LocalStore::queryValidDerivers(const Path & path) } +PathSet LocalStore::queryDerivationOutputs(const Path & path) +{ + SQLiteTxn txn(db); + + SQLiteStmtUse use(stmtQueryDerivationOutputs); + stmtQueryDerivationOutputs.bind(queryPathInfo(path).id); + + PathSet outputs; + int r; + while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); + assert(s); + outputs.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, format("error getting outputs of `%1%'") % path); + + return outputs; +} + + void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) { if (run.pid != -1) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6bd47b305..1a4acbe0e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -109,6 +109,8 @@ public: derivation that was actually used to produce `path', which may not exist anymore.) */ PathSet queryValidDerivers(const Path & path); + + PathSet queryDerivationOutputs(const Path & path); PathSet querySubstitutablePaths(); @@ -206,6 +208,7 @@ private: SQLiteStmt stmtHasPathFailed; SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; int getSchema(); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 2d7d13a0e..f79cb11cc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -31,10 +31,10 @@ void computeFSClosure(const Path & storePath, store->queryReferences(storePath, references); if (includeOutputs && isDerivation(storePath)) { - Derivation drv = derivationFromPath(storePath); - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (store->isValidPath(i->second.path)) - computeFSClosure(i->second.path, paths, flipDirection, true); + PathSet outputs = store->queryDerivationOutputs(storePath); + foreach (PathSet::iterator, i, outputs) + if (store->isValidPath(*i)) + computeFSClosure(*i, paths, flipDirection, true); } foreach (PathSet::iterator, i, references) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5143143f5..07cb62dc8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -294,6 +294,12 @@ Path RemoteStore::queryDeriver(const Path & path) } +PathSet RemoteStore::queryDerivationOutputs(const Path & path) +{ + throw Error("not yet implemented"); +} + + Path RemoteStore::addToStore(const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3d55d23d9..8bab1d8c4 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -37,6 +37,8 @@ public: Path queryDeriver(const Path & path); + PathSet queryDerivationOutputs(const Path & path); + bool hasSubstitutes(const Path & path); bool querySubstitutablePathInfo(const Path & path, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d85ae0c9a..b6a8ff40e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -139,6 +139,9 @@ public: no deriver has been set. */ virtual Path queryDeriver(const Path & path) = 0; + /* Query the outputs of the derivation denoted by `path'. */ + virtual PathSet queryDerivationOutputs(const Path & path) = 0; + /* Query whether a path has substitutes. */ virtual bool hasSubstitutes(const Path & path) = 0; From 9cda61694957f2f0428779319f85f626578d0cf0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 14:18:55 +0000 Subject: [PATCH 017/128] * The database needs a trigger to get rid of self-references to prevent a foreign key constraint violation on the Refs table when deleting a path. --- src/libstore/local-store.cc | 12 ++++++++++-- src/libstore/schema.sql | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0823e785b..7df67555e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -109,7 +109,11 @@ struct SQLiteStmtUse } ~SQLiteStmtUse() { - stmt.reset(); + try { + stmt.reset(); + } catch (...) { + ignoreException(); + } } }; @@ -798,6 +802,8 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) void LocalStore::invalidatePath(const Path & path) { debug(format("invalidating path `%1%'") % path); + + SQLiteTxn txn(db); SQLiteStmtUse use(stmtInvalidatePath); @@ -807,7 +813,9 @@ void LocalStore::invalidatePath(const Path & path) throw SQLiteError(db, format("invalidating path `%1%' in database") % path); /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ + care of deleting the references entries for `path'. */ + + txn.commit(); } diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 682ce5ed7..4adad34b7 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -17,6 +17,16 @@ create table if not exists Refs ( create index if not exists IndexReferrer on Refs(referrer); create index if not exists IndexReference on Refs(reference); +-- Paths can refer to themselves, causing a tuple (N, N) in the Refs +-- table. This causes a deletion of the corresponding row in +-- ValidPaths to cause a foreign key constraint violation (due to `on +-- delete restrict' on the `reference' column). Therefore, explicitly +-- get rid of self-references. +create trigger DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; + create table if not exists DerivationOutputs ( drv integer not null, id text not null, -- symbolic output id, usually "out" From 2b20318b0e968432438a7528b2d11d05585877c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 14:24:37 +0000 Subject: [PATCH 018/128] --- src/libstore/local-store.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7df67555e..7db2aabe4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -803,8 +803,6 @@ void LocalStore::invalidatePath(const Path & path) { debug(format("invalidating path `%1%'") % path); - SQLiteTxn txn(db); - SQLiteStmtUse use(stmtInvalidatePath); stmtInvalidatePath.bind(path); @@ -814,8 +812,6 @@ void LocalStore::invalidatePath(const Path & path) /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ - - txn.commit(); } From b4e6d98fc33df055c8230903bb7d9633042a6374 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Feb 2010 22:12:46 +0000 Subject: [PATCH 019/128] * configure: flag --with-sqlite. --- configure.ac | 20 +++++++++++++++++--- src/libstore/Makefile.am | 5 +++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index f726b1ae4..9d3d9f76e 100644 --- a/configure.ac +++ b/configure.ac @@ -245,13 +245,27 @@ AC_SUBST(bzip2_include) AC_SUBST(bzip2_bin) AC_SUBST(bzip2_bin_test) +AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], + [prefix of SQLite]), + sqlite=$withval, sqlite=) +AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") +if test -z "$sqlite"; then + sqlite_lib='-L${top_builddir}/externals/inst-sqlite/lib -lsqlite3' + sqlite_include='-I${top_builddir}/externals/inst-sqlite/include' + sqlite_bin='${top_builddir}/externals/inst-sqlite/bin' +else + sqlite_lib="-L$sqlite/lib -lsqlite3" + sqlite_include="-I$sqlite/include" + sqlite_bin="$sqlite/bin" +fi +AC_SUBST(sqlite_lib) +AC_SUBST(sqlite_include) +AC_SUBST(sqlite_bin) + AC_CHECK_LIB(pthread, pthread_mutex_init) -LDFLAGS="-lsqlite3" - - AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state], [do not initialise DB etc. in `make install']), init_state=$enableval, init_state=yes) diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 0140857b8..2f8bb72da 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -10,14 +10,15 @@ pkginclude_HEADERS = \ globals.hh references.hh pathlocks.hh \ worker-protocol.hh -libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ +libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ + ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ BUILT_SOURCES = derivations-ast.cc derivations-ast.hh EXTRA_DIST = derivations-ast.def derivations-ast.cc AM_CXXFLAGS = -Wall \ - -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil + -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil local-store.lo: schema.sql.hh From 63b09c5e414f9592f52d4f3a19185df6f8c00e42 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Feb 2010 22:31:38 +0000 Subject: [PATCH 020/128] --- src/libstore/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 2f8bb72da..e7d683499 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -15,7 +15,7 @@ libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ BUILT_SOURCES = derivations-ast.cc derivations-ast.hh -EXTRA_DIST = derivations-ast.def derivations-ast.cc +EXTRA_DIST = derivations-ast.def derivations-ast.cc schema.sql AM_CXXFLAGS = -Wall \ -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil From dc6d1ec67ea7af4e509c171852620befb9184441 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Feb 2010 23:24:49 +0000 Subject: [PATCH 021/128] --- release.nix | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/release.nix b/release.nix index e9afe9c9e..f5b0f3563 100644 --- a/release.nix +++ b/release.nix @@ -77,34 +77,11 @@ let configureFlags = '' --disable-init-state - --with-aterm=${aterm} --with-bzip2=${bzip2} + --with-aterm=${aterm} --with-bzip2=${bzip2} --with-sqlite=${sqlite} ''; }; - /* - static = - { tarball ? jobs.tarball {} - , system ? "i686-linux" - }: - - with import nixpkgs {inherit system;}; - - releaseTools.binaryTarball { - name = "nix-static-tarball"; - src = tarball; - - buildInputs = [curl perl bzip2]; - - configureFlags = '' - --disable-init-state - --with-aterm=${aterm} --with-bzip2=${bzip2} - --enable-static-nix - ''; - }; - */ - - coverage = { tarball ? jobs.tarball {} }: @@ -123,7 +100,7 @@ let configureFlags = '' --disable-init-state --disable-shared - --with-aterm=${aterm} --with-bzip2=${bzip2} + --with-aterm=${aterm} --with-bzip2=${bzip2} --with-sqlite=${sqlite} ''; lcovFilter = ["*/boost/*" "*-tab.*"]; From 8520de4720a798701a41b6ab3e6cf8d6cc71caa3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 10:48:57 +0000 Subject: [PATCH 022/128] * Include the sqlite tarball in the Nix tarball. --- externals/Makefile.am | 54 +++++++++++++++++++++++++++++++------------ release.nix | 3 +++ 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/externals/Makefile.am b/externals/Makefile.am index c9bc05d71..3a0b77609 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -14,14 +14,10 @@ $(ATERM): $(ATERM).tar.gz patch -d $(ATERM) -p1 < ./max-long.patch patch -d $(ATERM) -p1 < ./sizeof.patch -have-aterm: - $(MAKE) $(ATERM) - touch have-aterm - if HAVE_ATERM build-aterm: else -build-aterm: have-aterm +build-aterm: $(ATERM) (pfx=`pwd` && \ cd $(ATERM) && \ CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm \ @@ -47,14 +43,10 @@ $(BZIP2).tar.gz: $(BZIP2): $(BZIP2).tar.gz gunzip < $(srcdir)/$(BZIP2).tar.gz | tar xvf - -have-bzip2: - $(MAKE) $(BZIP2) - touch have-bzip2 - if HAVE_BZIP2 build-bzip2: else -build-bzip2: have-bzip2 +build-bzip2: $(BZIP2) (pfx=`pwd` && \ cd $(BZIP2) && \ $(MAKE) && \ @@ -67,10 +59,42 @@ install: endif -all: build-aterm build-bzip2 +# SQLite -EXTRA_DIST = $(ATERM).tar.gz $(BZIP2).tar.gz max-long.patch sizeof.patch +SQLITE_VERSION = 3.6.22 +SQLITE = sqlite-$(SQLITE_VERSION) +SQLITE_TAR = sqlite-amalgamation-$(SQLITE_VERSION).tar.gz -ext-clean: - $(RM) -f have-aterm build-aterm have-bzip2 build-bzip2 - $(RM) -rf $(ATERM) $(BZIP2) +$(SQLITE_TAR): + @echo "Nix requires the SQLite library to build." + @echo "Please download version $(SQLITE_VERSION) from" + @echo " http://www.sqlite.org/$(SQLITE_TAR)" + @echo "and place it in the externals/ directory." + false + +$(SQLITE): $(SQLITE_TAR) + gzip -d < $(srcdir)/$(SQLITE_TAR) | tar xvf - + +if HAVE_SQLITE +build-sqlite: +else +build-sqlite: $(SQLITE) + (pfx=`pwd` && \ + cd $(SQLITE) && \ + CC="$(CC)" ./configure --prefix=$$pfx/inst-sqlite \ + --disable-shared --enable-static && \ + $(MAKE) && \ + $(MAKE) check && \ + $(MAKE) install) + touch build-sqlite +endif + + +all: build-aterm build-bzip2 build-sqlite + +EXTRA_DIST = $(ATERM).tar.gz $(BZIP2).tar.gz $(SQLITE_TAR) max-long.patch sizeof.patch + +clean: + $(RM) -f build-aterm build-bzip2 build-sqlite + $(RM) -rf $(ATERM) $(BZIP2) $(SQLITE) + $(RM) -rf inst-aterm inst-bzip2 inst-sqlite diff --git a/release.nix b/release.nix index f5b0f3563..0de1b7bb2 100644 --- a/release.nix +++ b/release.nix @@ -37,6 +37,9 @@ let stripHash ${bzip2.src} cp -pv ${bzip2.src} externals/$strippedName + stripHash ${sqlite.src} + cp -pv ${sqlite.src} externals/$strippedName + # TeX needs a writable font cache. export VARTEXFONTS=$TMPDIR/texfonts ''; From 462bd50aefca3d70e76ccf17f42c2aef4714c95b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 10:57:57 +0000 Subject: [PATCH 023/128] * Use normal (rather than full) synchronous mode, which I gather from the description at http://www.sqlite.org/atomiccommit.html should be safe enough. --- src/libstore/local-store.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7db2aabe4..8e5448446 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -219,6 +219,10 @@ LocalStore::LocalStore() /* !!! check whether sqlite has been built with foreign key support */ + /* "Normal" synchronous mode should be safe enough. */ + if (sqlite3_exec(db, "pragma synchronous = normal;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "changing synchronous mode to normal"); + /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start the upgrade at the same time. */ From 69d9df7fe6a0c4043e6f913137b5322bc5c48073 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 11:04:03 +0000 Subject: [PATCH 024/128] * Don't fork so much. --- tests/referrers.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/referrers.sh b/tests/referrers.sh index 752f05c3d..8f4233b99 100644 --- a/tests/referrers.sh +++ b/tests/referrers.sh @@ -1,8 +1,5 @@ source common.sh -# This takes way to long on Cygwin (because process creation is so slow...). -if test "$system" = i686-cygwin; then exit 0; fi - clearStore max=2500 @@ -15,12 +12,12 @@ echo "making registration..." for ((n = 0; n < $max; n++)); do storePath=$NIX_STORE_DIR/$n - touch $storePath + echo -n > $storePath ref2=$NIX_STORE_DIR/$((n+1)) if test $((n+1)) = $max; then ref2=$reference fi - (echo $storePath && echo && echo 2 && echo $reference && echo $ref2) + echo $storePath; echo; echo 2; echo $reference; echo $ref2 done > $TEST_ROOT/reg_info echo "registering..." From bb82310dba8a70b539122db20712fbd8fd9e840e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 11:06:17 +0000 Subject: [PATCH 025/128] * Set the path to sqlite3 properly. --- substitute.mk | 1 + tests/common.sh.in | 1 + 2 files changed, 2 insertions(+) diff --git a/substitute.mk b/substitute.mk index ae11ce2b3..430af16bf 100644 --- a/substitute.mk +++ b/substitute.mk @@ -24,6 +24,7 @@ -e "s^@xmlflags\@^$(xmlflags)^g" \ -e "s^@xsltproc\@^$(xsltproc)^g" \ -e "s^@aterm_bin\@^$(aterm_bin)^g" \ + -e "s^@sqlite_bin\@^$(sqlite_bin)^g" \ -e "s^@version\@^$(VERSION)^g" \ -e "s^@testPath\@^$(coreutils):$$(dirname $$(type -P expr))^g" \ < $< > $@ || rm $@ diff --git a/tests/common.sh.in b/tests/common.sh.in index 85dd3a389..35e17c3bf 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -39,6 +39,7 @@ export dot=@dot@ export xmllint="@xmllint@" export xmlflags="@xmlflags@" export xsltproc="@xsltproc@" +export sqlite3="@sqlite_bin@/bin/sqlite3" export SHELL="@shell@" # Hack to get "atdiff" to run on Cygwin (Windows looks for From 5954eadf6741e0b4cdab23c414d7a6b470cd06db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:16:50 +0000 Subject: [PATCH 026/128] * Remove the fdatasync check since it's no longer needed. --- configure.ac | 1 - src/libutil/util.cc | 3 +-- src/libutil/util.hh | 6 +----- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 9d3d9f76e..d5db92af3 100644 --- a/configure.ac +++ b/configure.ac @@ -279,7 +279,6 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) # Nice to have, but not essential. AC_CHECK_FUNCS([strsignal]) AC_CHECK_FUNCS([posix_fallocate]) -AC_CHECK_FUNCS([fdatasync]) # This is needed if ATerm or bzip2 are static libraries, diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8c52625a2..3f76be670 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -227,13 +227,12 @@ string readFile(const Path & path) } -void writeFile(const Path & path, const string & s, bool doFsync) +void writeFile(const Path & path, const string & s) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); if (fd == -1) throw SysError(format("opening file `%1%'") % path); writeFull(fd, (unsigned char *) s.c_str(), s.size()); - if (doFsync) fdatasync(fd); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 90132da73..a3580b547 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -12,10 +12,6 @@ #include -#ifndef HAVE_FDATASYNC -#define fdatasync fsync -#endif - namespace nix { @@ -66,7 +62,7 @@ string readFile(int fd); string readFile(const Path & path); /* Write a string to a file. */ -void writeFile(const Path & path, const string & s, bool doFsync = false); +void writeFile(const Path & path, const string & s); /* Read a line from a file descriptor. */ string readLine(int fd); From fa6a4fcb11fa6db1c35cd1199ce8d1dc794e6cd4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:18:48 +0000 Subject: [PATCH 027/128] * Add ${sqlite_lib} everywhere. Just adding it in `libstore' doesn't work on x86_64 when sqlite is compiled statically. --- src/libstore/Makefile.am | 2 +- src/nix-env/Makefile.am | 2 +- src/nix-hash/Makefile.am | 2 +- src/nix-instantiate/Makefile.am | 2 +- src/nix-store/Makefile.am | 2 +- src/nix-worker/Makefile.am | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index e7d683499..b19238e7d 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -11,7 +11,7 @@ pkginclude_HEADERS = \ worker-protocol.hh libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ - ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + @ADDITIONAL_NETWORK_LIBS@ BUILT_SOURCES = derivations-ast.cc derivations-ast.hh diff --git a/src/nix-env/Makefile.am b/src/nix-env/Makefile.am index 900524f76..35585602e 100644 --- a/src/nix-env/Makefile.am +++ b/src/nix-env/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-env nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-env.o: help.txt.hh diff --git a/src/nix-hash/Makefile.am b/src/nix-hash/Makefile.am index 350aa8ebd..11d2a2d8d 100644 --- a/src/nix-hash/Makefile.am +++ b/src/nix-hash/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash nix_hash_SOURCES = nix-hash.cc help.txt nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-hash.o: help.txt.hh diff --git a/src/nix-instantiate/Makefile.am b/src/nix-instantiate/Makefile.am index 3f6671719..6b9a95943 100644 --- a/src/nix-instantiate/Makefile.am +++ b/src/nix-instantiate/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate nix_instantiate_SOURCES = nix-instantiate.cc help.txt nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-instantiate.o: help.txt.hh diff --git a/src/nix-store/Makefile.am b/src/nix-store/Makefile.am index ca0fec570..d604920dd 100644 --- a/src/nix-store/Makefile.am +++ b/src/nix-store/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-store nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-store.o: help.txt.hh diff --git a/src/nix-worker/Makefile.am b/src/nix-worker/Makefile.am index d1163ce37..a54e5ec21 100644 --- a/src/nix-worker/Makefile.am +++ b/src/nix-worker/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker nix_worker_SOURCES = nix-worker.cc help.txt nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-worker.o: help.txt.hh From fae0427324269e420d8ea5774f2fab2330acda9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:25:48 +0000 Subject: [PATCH 028/128] * ADDITIONAL_NETWORK_LIBS -> LIBS. --- configure.ac | 3 +-- src/libstore/Makefile.am | 3 +-- src/nix-env/Makefile.am | 2 +- src/nix-hash/Makefile.am | 2 +- src/nix-instantiate/Makefile.am | 2 +- src/nix-store/Makefile.am | 2 +- src/nix-worker/Makefile.am | 2 +- 7 files changed, 7 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index d5db92af3..df6508fcb 100644 --- a/configure.ac +++ b/configure.ac @@ -79,8 +79,7 @@ fi # Solaris-specific stuff. if test "$sys_name" = "sunos"; then # Solaris requires -lsocket -lnsl for network functions - ADDITIONAL_NETWORK_LIBS="-lsocket -lnsl" - AC_SUBST(ADDITIONAL_NETWORK_LIBS) + LIBS="-lsocket -lnsl $LIBS" fi AC_PROG_CC diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index b19238e7d..dbfc1474e 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -10,8 +10,7 @@ pkginclude_HEADERS = \ globals.hh references.hh pathlocks.hh \ worker-protocol.hh -libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ - @ADDITIONAL_NETWORK_LIBS@ +libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la BUILT_SOURCES = derivations-ast.cc derivations-ast.hh diff --git a/src/nix-env/Makefile.am b/src/nix-env/Makefile.am index 35585602e..07b29c186 100644 --- a/src/nix-env/Makefile.am +++ b/src/nix-env/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-env nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-env.o: help.txt.hh diff --git a/src/nix-hash/Makefile.am b/src/nix-hash/Makefile.am index 11d2a2d8d..d7c569b68 100644 --- a/src/nix-hash/Makefile.am +++ b/src/nix-hash/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash nix_hash_SOURCES = nix-hash.cc help.txt nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-hash.o: help.txt.hh diff --git a/src/nix-instantiate/Makefile.am b/src/nix-instantiate/Makefile.am index 6b9a95943..f39331805 100644 --- a/src/nix-instantiate/Makefile.am +++ b/src/nix-instantiate/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate nix_instantiate_SOURCES = nix-instantiate.cc help.txt nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-instantiate.o: help.txt.hh diff --git a/src/nix-store/Makefile.am b/src/nix-store/Makefile.am index d604920dd..0d0d0e033 100644 --- a/src/nix-store/Makefile.am +++ b/src/nix-store/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-store nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-store.o: help.txt.hh diff --git a/src/nix-worker/Makefile.am b/src/nix-worker/Makefile.am index a54e5ec21..82bbaf1f7 100644 --- a/src/nix-worker/Makefile.am +++ b/src/nix-worker/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker nix_worker_SOURCES = nix-worker.cc help.txt nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-worker.o: help.txt.hh From 90b6352d0a5d08dc7feabcfde92653dd1f6e324b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:48:00 +0000 Subject: [PATCH 029/128] * Do registerValidPaths() in one transaction, which is much faster. E.g. it cuts the runtime of the referrers test from 50s to 23s. --- src/libstore/local-store.cc | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8e5448446..6e8082776 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -765,39 +765,19 @@ Hash LocalStore::queryPathHash(const Path & path) } -static void dfsVisit(std::map & infos, - const Path & path, PathSet & visited, Paths & sorted) -{ - if (visited.find(path) != visited.end()) return; - visited.insert(path); - - ValidPathInfo & info(infos[path]); - - foreach (PathSet::iterator, i, info.references) - if (infos.find(*i) != infos.end()) - dfsVisit(infos, *i, visited, sorted); - - sorted.push_back(path); -} - - void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - std::map infosMap; + SQLiteTxn txn(db); - /* Sort the paths topologically under the references relation, so - that if path A is referenced by B, then A is registered before - B. */ - foreach (ValidPathInfos::const_iterator, i, infos) - infosMap[i->path] = *i; + foreach (ValidPathInfos::const_iterator, i, infos) addValidPath(*i); - PathSet visited; - Paths sorted; - foreach (ValidPathInfos::const_iterator, i, infos) - dfsVisit(infosMap, i->path, visited, sorted); + foreach (ValidPathInfos::const_iterator, i, infos) { + unsigned long long referrer = queryPathInfo(i->path).id; + foreach (PathSet::iterator, j, i->references) + addReference(referrer, queryPathInfo(*j).id); + } - foreach (Paths::iterator, i, sorted) - registerValidPath(infosMap[*i]); + txn.commit(); } From a3c63d0d6c2570ce3218be809f67dddc5239cdff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 13:12:57 +0000 Subject: [PATCH 030/128] * Disable fsync() in SQLite if the fsync-metadata option is set to false. * Change the default for `fsync-metadata' to true. * Disable `fsync-metadata' in `make check'. --- doc/manual/conf-file.xml | 2 +- src/libstore/local-store.cc | 12 +++++++----- src/libstore/local-store.hh | 3 --- tests/init.sh | 1 + 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 2ee268097..19e86808e 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -241,7 +241,7 @@ build-use-chroot = /dev /proc /bin Nix store metadata (in /nix/var/nix/db) are synchronously flushed to disk. This improves robustness in case of system crashes, but reduces performance. The default is - false. + true. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6e8082776..67a91bb49 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -219,9 +219,13 @@ LocalStore::LocalStore() /* !!! check whether sqlite has been built with foreign key support */ - /* "Normal" synchronous mode should be safe enough. */ - if (sqlite3_exec(db, "pragma synchronous = normal;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "changing synchronous mode to normal"); + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; + if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting synchronous mode"); /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start @@ -243,8 +247,6 @@ LocalStore::LocalStore() "please upgrade Nix to version 0.12 first."); else if (curSchema < 6) upgradeStore6(); else prepareStatements(); - - doFsync = queryBoolSetting("fsync-metadata", false); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1a4acbe0e..ec0b482ea 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -191,9 +191,6 @@ private: /* Lock file used for upgrading. */ AutoCloseFD globalLock; - /* Whether to do an fsync() after writing Nix metadata. */ - bool doFsync; - /* The SQLite database object. */ SQLite db; diff --git a/tests/init.sh b/tests/init.sh index 691cb669b..64947031b 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -40,6 +40,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < Date: Wed, 24 Feb 2010 13:13:39 +0000 Subject: [PATCH 031/128] --- tests/referrers.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/referrers.sh b/tests/referrers.sh index 8f4233b99..a0c195d5a 100644 --- a/tests/referrers.sh +++ b/tests/referrers.sh @@ -2,7 +2,7 @@ source common.sh clearStore -max=2500 +max=500 reference=$NIX_STORE_DIR/abcdef touch $reference @@ -22,11 +22,11 @@ done > $TEST_ROOT/reg_info echo "registering..." -time $nixstore --register-validity < $TEST_ROOT/reg_info +$nixstore --register-validity < $TEST_ROOT/reg_info echo "collecting garbage..." ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref -time $nixstore --gc +$nixstore --gc if test "$(sqlite3 ./test-tmp/db/db.sqlite 'select count(*) from Refs')" -ne 0; then echo "referrers not cleaned up" From fefd467539683e45e3682a54cefc67ead33eb346 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 13:24:27 +0000 Subject: [PATCH 032/128] * `helpText' is now zero-terminated. --- src/nix-env/nix-env.cc | 2 +- src/nix-hash/nix-hash.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix-worker/nix-worker.cc | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 35caf687b..d11be3808 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -70,7 +70,7 @@ typedef void (* Operation) (Globals & globals, void printHelp() { - cout << string((char *) helpText, sizeof helpText); + cout << string((char *) helpText); } diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc index 1282af070..a70f59e0c 100644 --- a/src/nix-hash/nix-hash.cc +++ b/src/nix-hash/nix-hash.cc @@ -10,7 +10,7 @@ using namespace nix; void printHelp() { - std::cout << string((char *) helpText, sizeof helpText); + std::cout << string((char *) helpText); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 3822de5c6..22df2fbe5 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -19,7 +19,7 @@ using namespace nix; void printHelp() { - std::cout << string((char *) helpText, sizeof helpText); + std::cout << string((char *) helpText); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 22effc65d..ddf2062c2 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -21,7 +21,7 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs); void printHelp() { - cout << string((char *) helpText, sizeof helpText); + cout << string((char *) helpText); } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index bd2209c6d..362b9b928 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -765,7 +765,7 @@ void run(Strings args) void printHelp() { - std::cout << string((char *) helpText, sizeof helpText); + std::cout << string((char *) helpText); } From 9fd85c94de90ec91ccf8a4bf04e8911e6e3e483b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 14:22:34 +0000 Subject: [PATCH 033/128] * Use `truncate' journal mode, which should be a bit faster. --- src/libstore/local-store.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 67a91bb49..6963ba5a1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -227,6 +227,10 @@ LocalStore::LocalStore() if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting synchronous mode"); + /* Use `truncate' journal mode, which should be a bit faster. */ + if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting journal mode"); + /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start the upgrade at the same time. */ From 6baa2a2f5ef5b11859b95f453338efaf7c5b2724 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 14:39:52 +0000 Subject: [PATCH 034/128] --- src/libstore/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 4adad34b7..7438632ed 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -22,7 +22,7 @@ create index if not exists IndexReference on Refs(reference); -- ValidPaths to cause a foreign key constraint violation (due to `on -- delete restrict' on the `reference' column). Therefore, explicitly -- get rid of self-references. -create trigger DeleteSelfRefs before delete on ValidPaths +create trigger if not exists DeleteSelfRefs before delete on ValidPaths begin delete from Refs where referrer = old.id and reference = old.id; end; From cfe742cfc50e40b590e75200179013dd62b07bde Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 15:07:23 +0000 Subject: [PATCH 035/128] * A function to query just the database id of a valid path. --- src/libstore/local-store.cc | 36 ++++++++++++++++++++---------------- src/libstore/local-store.hh | 2 ++ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6963ba5a1..bfb253bc1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -461,10 +461,8 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) unsigned long long id = addValidPath(info2); - foreach (PathSet::const_iterator, i, info2.references) { - ValidPathInfo ref = queryPathInfo(*i); - addReference(id, ref.id); - } + foreach (PathSet::const_iterator, i, info2.references) + addReference(id, queryValidPathId(*i)); txn.commit(); } @@ -551,6 +549,17 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } +unsigned long long LocalStore::queryValidPathId(const Path & path) +{ + SQLiteStmtUse use(stmtQueryPathInfo); + stmtQueryPathInfo.bind(path); + int res = sqlite3_step(stmtQueryPathInfo); + if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); + if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + throw SQLiteError(db, "querying path in database"); +} + + bool LocalStore::isValidPath(const Path & path) { SQLiteStmtUse use(stmtQueryPathInfo); @@ -644,7 +653,7 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) SQLiteTxn txn(db); SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryPathInfo(path).id); + stmtQueryDerivationOutputs.bind(queryValidPathId(path)); PathSet outputs; int r; @@ -778,9 +787,9 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) foreach (ValidPathInfos::const_iterator, i, infos) addValidPath(*i); foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryPathInfo(i->path).id; + unsigned long long referrer = queryValidPathId(i->path); foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryPathInfo(*j).id); + addReference(referrer, queryValidPathId(*j)); } txn.commit(); @@ -1229,11 +1238,8 @@ void LocalStore::upgradeStore6() SQLiteTxn txn(db); - std::map pathToId; - foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfoOld(*i); - pathToId[*i] = addValidPath(info); + addValidPath(queryPathInfoOld(*i)); std::cerr << "."; } @@ -1241,11 +1247,9 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfoOld(*i); - foreach (PathSet::iterator, j, info.references) { - if (pathToId.find(*j) == pathToId.end()) - throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); - addReference(pathToId[*i], pathToId[*j]); - } + unsigned long long referrer = queryValidPathId(*i); + foreach (PathSet::iterator, j, info.references) + addReference(referrer, queryValidPathId(*j)); std::cerr << "."; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ec0b482ea..1e4080bec 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -213,6 +213,8 @@ private: void prepareStatements(); + unsigned long long queryValidPathId(const Path & path); + unsigned long long addValidPath(const ValidPathInfo & info); void addReference(unsigned long long referrer, unsigned long long reference); From e33f67ff0b02cc8fc2e0024dd87a6d61467df177 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 16:30:20 +0000 Subject: [PATCH 036/128] * Refactor the upgrade / database initialisation logic a bit. --- src/libstore/local-store.cc | 121 +++++++++++++++++++----------------- src/libstore/local-store.hh | 4 +- 2 files changed, 64 insertions(+), 61 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bfb253bc1..7e205a9bf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -205,52 +205,43 @@ LocalStore::LocalStore() lockFile(globalLock, ltRead, true); } - /* Open the Nix database. */ - if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) - throw Error("cannot open SQLite database"); - - if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) - throw SQLiteError(db, "setting timeout"); - - if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "enabling foreign keys"); - - /* !!! check whether sqlite has been built with foreign key - support */ - - /* Whether SQLite should fsync(). "Normal" synchronous mode - should be safe enough. If the user asks for it, don't sync at - all. This can cause database corruption if the system - crashes. */ - string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; - if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting synchronous mode"); - - /* Use `truncate' journal mode, which should be a bit faster. */ - if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting journal mode"); - /* Check the current database schema and if necessary do an - upgrade. !!! Race condition: several processes could start - the upgrade at the same time. */ + upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") % curSchema % nixSchemaVersion); - if (curSchema == 0) { /* new store */ + + else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - initSchema(); + openDB(true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } - else if (curSchema == 1) throw Error("your Nix store is no longer supported"); - else if (curSchema < 5) - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); - else if (curSchema < 6) upgradeStore6(); - else prepareStatements(); + + else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + + if (!lockFile(globalLock, ltWrite, false)) { + printMsg(lvlError, "waiting for exclusive access to the Nix store..."); + lockFile(globalLock, ltWrite, true); + } + + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); + + if (curSchema < 6) upgradeStore6(); + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock, ltRead, true); + } + + else openDB(false); } @@ -280,19 +271,42 @@ int LocalStore::getSchema() } -void LocalStore::initSchema() +void LocalStore::openDB(bool create) { + /* Open the Nix database. */ + if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + throw Error("cannot open SQLite database"); + + if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + throw SQLiteError(db, "setting timeout"); + + if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "enabling foreign keys"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; + if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting synchronous mode"); + + /* Use `truncate' journal mode, which should be a bit faster. */ + if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting journal mode"); + + /* Initialise the database schema, if necessary. */ + if (create) { #include "schema.sql.hh" + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "initialising database schema"); + } - if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "initialising database schema"); - - prepareStatements(); -} - - -void LocalStore::prepareStatements() -{ + /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, "insert or replace into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, @@ -1225,14 +1239,9 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) /* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ void LocalStore::upgradeStore6() { - if (!lockFile(globalLock, ltWrite, false)) { - printMsg(lvlError, "waiting for exclusive access to the Nix store..."); - lockFile(globalLock, ltWrite, true); - } - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - initSchema(); + openDB(true); PathSet validPaths = queryValidPathsOld(); @@ -1256,10 +1265,6 @@ void LocalStore::upgradeStore6() std::cerr << "\n"; txn.commit(); - - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - - lockFile(globalLock, ltRead, true); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1e4080bec..0c5f04158 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -209,9 +209,7 @@ private: int getSchema(); - void initSchema(); - - void prepareStatements(); + void openDB(bool create); unsigned long long queryValidPathId(const Path & path); From af565c348a286fb55ad17f8c3c4233465d32a9f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 16:44:43 +0000 Subject: [PATCH 037/128] * Support read-only access to the database. --- src/libstore/local-store.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7e205a9bf..95e775ce7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -173,7 +173,10 @@ LocalStore::LocalStore() schemaPath = nixDBPath + "/schema"; - if (readOnlyMode) return; + if (readOnlyMode) { + openDB(false); + return; + } /* Create missing state directories if they don't already exist. */ createDirs(nixStore); @@ -197,6 +200,7 @@ LocalStore::LocalStore() } catch (SysError & e) { if (e.errNo != EACCES) throw; readOnlyMode = true; + openDB(false); return; } From e42401ee7b0c0b166677595d4fd1255176ba33c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Feb 2010 15:52:22 +0000 Subject: [PATCH 038/128] * Implement RemoteStore::queryDerivationOutputs(). --- src/libstore/remote-store.cc | 6 +++++- src/libstore/worker-protocol.hh | 1 + src/nix-worker/nix-worker.cc | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 07cb62dc8..b438c074f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -296,7 +296,11 @@ Path RemoteStore::queryDeriver(const Path & path) PathSet RemoteStore::queryDerivationOutputs(const Path & path) { - throw Error("not yet implemented"); + openConnection(); + writeInt(wopQueryDerivationOutputs, to); + writeString(path, to); + processStderr(); + return readStorePaths(from); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index e44c1e36b..9e36aab0f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -34,6 +34,7 @@ typedef enum { wopSetOptions = 19, wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, } WorkerOp; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 362b9b928..59ab48776 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -313,14 +313,16 @@ static void performOp(unsigned int clientVersion, } case wopQueryReferences: - case wopQueryReferrers: { + case wopQueryReferrers: + case wopQueryDerivationOutputs: { Path path = readStorePath(from); startWork(); PathSet paths; if (op == wopQueryReferences) store->queryReferences(path, paths); - else + else if (op == wopQueryReferrers) store->queryReferrers(path, paths); + else paths = store->queryDerivationOutputs(path); stopWork(); writeStringSet(paths, to); break; From 24035b98b155dd2d35414bdf015b3d7ab9a57af7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2010 12:05:01 +0000 Subject: [PATCH 039/128] * Implement RemoteStore::queryValidPaths(). --- src/libstore/remote-store.cc | 4 +++- src/libstore/worker-protocol.hh | 1 + src/nix-worker/nix-worker.cc | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b438c074f..c5d7975b5 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -214,7 +214,9 @@ bool RemoteStore::isValidPath(const Path & path) PathSet RemoteStore::queryValidPaths() { openConnection(); - throw Error("not implemented"); + writeInt(wopQueryValidPaths, to); + processStderr(); + return readStorePaths(from); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 9e36aab0f..c3096010b 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -35,6 +35,7 @@ typedef enum { wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, wopQueryDerivationOutputs = 22, + wopQueryValidPaths = 23, } WorkerOp; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 59ab48776..a41fb2e15 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -520,6 +520,14 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryValidPaths: { + startWork(); + PathSet paths = store->queryValidPaths(); + stopWork(); + writeStringSet(paths, to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } From 966ffb29a7a7de00f3521da05f325ae8c7e8e35e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2010 13:10:57 +0000 Subject: [PATCH 040/128] * Update the Valgrind suppressions for ATerm 2.5. --- aterm-gc.supp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/aterm-gc.supp b/aterm-gc.supp index dcd6371fe..21b9a2372 100644 --- a/aterm-gc.supp +++ b/aterm-gc.supp @@ -115,3 +115,35 @@ fun:* fun:AT_collect } + +{ + ATerm library conservatively scans for GC roots + Memcheck:Value4 + fun:* + fun:* + fun:mark_phase +} + +{ + ATerm library conservatively scans for GC roots + Memcheck:Cond + fun:* + fun:* + fun:mark_phase +} + +{ + ATerm library conservatively scans for GC roots + Memcheck:Value4 + fun:* + fun:* + fun:mark_phase_young +} + +{ + ATerm library conservatively scans for GC roots + Memcheck:Cond + fun:* + fun:* + fun:mark_phase_young +} From 594eaddd1157db8abe2e1c47cdf2180f027559ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 15:58:13 +0000 Subject: [PATCH 041/128] * When using the included sqlite/aterm libraries, build with --enable-shared. * In libutil/libstore/libexpr etc., link against sqlite and aterm. * Some more header file hygiene. --- configure.ac | 16 +++++++++------ externals/Makefile.am | 28 +++++++++++++------------- src/libexpr/attr-path.cc | 1 + src/libexpr/common-opts.cc | 1 + src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 3 ++- src/libexpr/expr-to-xml.hh | 1 - src/libexpr/get-drvs.cc | 1 + src/libexpr/primops.cc | 1 + src/libstore/Makefile.am | 4 ++-- src/libstore/derivations.hh | 2 +- src/libutil/Makefile.am | 2 +- src/libutil/aterm-map.hh | 3 ++- src/nix-env/Makefile.am | 5 +++-- src/nix-env/nix-env.cc | 1 + src/nix-hash/Makefile.am | 2 +- src/nix-instantiate/Makefile.am | 2 +- src/nix-instantiate/nix-instantiate.cc | 1 + src/nix-setuid-helper/Makefile.am | 5 ++--- src/nix-store/Makefile.am | 4 ++-- src/nix-worker/Makefile.am | 4 ++-- 21 files changed, 50 insertions(+), 38 deletions(-) diff --git a/configure.ac b/configure.ac index df6508fcb..35562882c 100644 --- a/configure.ac +++ b/configure.ac @@ -197,9 +197,9 @@ AC_ARG_WITH(aterm, AC_HELP_STRING([--with-aterm=PATH], aterm=$withval, aterm=) AM_CONDITIONAL(HAVE_ATERM, test -n "$aterm") if test -z "$aterm"; then - aterm_lib='-L${top_builddir}/externals/inst-aterm/lib -lATerm' - aterm_include='-I${top_builddir}/externals/inst-aterm/include' - aterm_bin='${top_builddir}/externals/inst-aterm/bin' + aterm_lib='-L${top_builddir}/externals/aterm-2.5/aterm -lATerm' + aterm_include='-I${top_builddir}/externals/aterm-2.5/aterm' + aterm_bin='${top_builddir}/externals/aterm-2.5/utils' else aterm_lib="-L$aterm/lib -lATerm" aterm_include="-I$aterm/include" @@ -224,6 +224,8 @@ AC_ARG_WITH(bzip2, AC_HELP_STRING([--with-bzip2=PATH], [prefix of bzip2]), bzip2=$withval, bzip2=) AM_CONDITIONAL(HAVE_BZIP2, test -n "$bzip2") +ATERM_VERSION=2.5 +AC_SUBST(ATERM_VERSION) if test -z "$bzip2"; then # Headers and libraries will be used from the temporary installation # in externals/inst-bzip2. @@ -248,10 +250,12 @@ AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], [prefix of SQLite]), sqlite=$withval, sqlite=) AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") +SQLITE_VERSION=3.6.22 +AC_SUBST(SQLITE_VERSION) if test -z "$sqlite"; then - sqlite_lib='-L${top_builddir}/externals/inst-sqlite/lib -lsqlite3' - sqlite_include='-I${top_builddir}/externals/inst-sqlite/include' - sqlite_bin='${top_builddir}/externals/inst-sqlite/bin' + sqlite_lib='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)/libsqlite3.la' + sqlite_include='-I${top_builddir}/externals/sqlite-$(SQLITE_VERSION)' + sqlite_bin='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)' else sqlite_lib="-L$sqlite/lib -lsqlite3" sqlite_include="-I$sqlite/include" diff --git a/externals/Makefile.am b/externals/Makefile.am index 3a0b77609..06ee9d590 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -1,11 +1,11 @@ # CWI ATerm -ATERM = aterm-2.5 +ATERM = aterm-$(ATERM_VERSION) $(ATERM).tar.gz: @echo "Nix requires the CWI ATerm library to build." - @echo "Please download version 2.5 from" - @echo " http://nixos.org/tarballs/aterm-2.5.tar.gz" + @echo "Please download version $(ATERM_VERSION) from" + @echo " http://nixos.org/tarballs/aterm-$(ATERM_VERSION).tar.gz" @echo "and place it in the externals/ directory." false @@ -20,12 +20,13 @@ else build-aterm: $(ATERM) (pfx=`pwd` && \ cd $(ATERM) && \ - CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm \ - --disable-shared --enable-static && \ + CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm --libdir=${pkglibdir} && \ $(MAKE) && \ - $(MAKE) check && \ - $(MAKE) install) + $(MAKE) check) touch build-aterm + +install-exec-local:: + cd $(ATERM) && make install endif @@ -53,7 +54,7 @@ build-bzip2: $(BZIP2) $(MAKE) install PREFIX=$$pfx/inst-bzip2) touch build-bzip2 -install: +install-exec-local:: mkdir -p $(DESTDIR)${bzip2_bin} $(INSTALL_PROGRAM) $(bzip2_bin_test)/bzip2 $(bzip2_bin_test)/bunzip2 $(DESTDIR)${bzip2_bin} endif @@ -61,7 +62,6 @@ endif # SQLite -SQLITE_VERSION = 3.6.22 SQLITE = sqlite-$(SQLITE_VERSION) SQLITE_TAR = sqlite-amalgamation-$(SQLITE_VERSION).tar.gz @@ -81,12 +81,12 @@ else build-sqlite: $(SQLITE) (pfx=`pwd` && \ cd $(SQLITE) && \ - CC="$(CC)" ./configure --prefix=$$pfx/inst-sqlite \ - --disable-shared --enable-static && \ - $(MAKE) && \ - $(MAKE) check && \ - $(MAKE) install) + CC="$(CC)" ./configure --disable-static --prefix=$$pfx/inst-sqlite --libdir=${pkglibdir} && \ + $(MAKE) ) touch build-sqlite + +install-exec-local:: + cd $(SQLITE) && make install endif diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index e8e4c050c..092d9b1c2 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,7 @@ #include "attr-path.hh" #include "nixexpr-ast.hh" #include "util.hh" +#include "aterm.hh" namespace nix { diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 9e3f8f961..0ef488373 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -2,6 +2,7 @@ #include "../libmain/shared.hh" #include "util.hh" #include "parser.hh" +#include "aterm.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index cd9c64594..5a0e8bc27 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nixexpr-ast.hh" #include "globals.hh" +#include "aterm.hh" #define LocalNoInline(f) static f __attribute__((noinline)); f diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index fed6d3472..00d0e3564 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -3,9 +3,10 @@ #include -#include "aterm.hh" #include "nixexpr.hh" +typedef union _ATermList * ATermList; + namespace nix { diff --git a/src/libexpr/expr-to-xml.hh b/src/libexpr/expr-to-xml.hh index 36b8e4042..576a46fc3 100644 --- a/src/libexpr/expr-to-xml.hh +++ b/src/libexpr/expr-to-xml.hh @@ -5,7 +5,6 @@ #include #include "nixexpr.hh" -#include "aterm.hh" namespace nix { diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1442d7988..cd5a85e5b 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,6 +1,7 @@ #include "get-drvs.hh" #include "nixexpr-ast.hh" #include "util.hh" +#include "aterm.hh" namespace nix { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7dddc91a8..bf2752d0d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -8,6 +8,7 @@ #include "nixexpr-ast.hh" #include "parser.hh" #include "names.hh" +#include "aterm.hh" #include #include diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index dbfc1474e..dd6760554 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -10,14 +10,14 @@ pkginclude_HEADERS = \ globals.hh references.hh pathlocks.hh \ worker-protocol.hh -libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la +libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} BUILT_SOURCES = derivations-ast.cc derivations-ast.hh EXTRA_DIST = derivations-ast.def derivations-ast.cc schema.sql AM_CXXFLAGS = -Wall \ - -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil + -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil -I${top_srcdir}/externals/sqlite-3.6.22/ local-store.lo: schema.sql.hh diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 042f4738d..c3f579bee 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,7 +1,7 @@ #ifndef __DERIVATIONS_H #define __DERIVATIONS_H -#include +typedef union _ATerm * ATerm; #include "hash.hh" diff --git a/src/libutil/Makefile.am b/src/libutil/Makefile.am index bd0996543..f17236d29 100644 --- a/src/libutil/Makefile.am +++ b/src/libutil/Makefile.am @@ -3,7 +3,7 @@ pkglib_LTLIBRARIES = libutil.la libutil_la_SOURCES = util.cc hash.cc serialise.cc \ archive.cc aterm.cc aterm-map.cc xml-writer.cc -libutil_la_LIBADD = ../boost/format/libformat.la +libutil_la_LIBADD = ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} pkginclude_HEADERS = util.hh hash.hh serialise.hh \ archive.hh aterm.hh aterm-map.hh xml-writer.hh types.hh diff --git a/src/libutil/aterm-map.hh b/src/libutil/aterm-map.hh index b732453a7..bcfefd9ee 100644 --- a/src/libutil/aterm-map.hh +++ b/src/libutil/aterm-map.hh @@ -1,7 +1,8 @@ #ifndef __ATERM_MAP_H #define __ATERM_MAP_H -#include +typedef union _ATerm * ATerm; + #include diff --git a/src/nix-env/Makefile.am b/src/nix-env/Makefile.am index 07b29c186..53fea3d9d 100644 --- a/src/nix-env/Makefile.am +++ b/src/nix-env/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-env nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-env.o: help.txt.hh @@ -11,6 +11,7 @@ nix-env.o: help.txt.hh ../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1) AM_CXXFLAGS = \ - -I$(srcdir)/.. ${aterm_include} \ + ${aterm_include} \ + -I$(srcdir)/.. \ -I$(srcdir)/../libutil -I$(srcdir)/../libstore \ -I$(srcdir)/../libexpr -I$(srcdir)/../libmain -I../libexpr diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d11be3808..d3f74c7ed 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -14,6 +14,7 @@ #include "xml-writer.hh" #include "store-api.hh" #include "util.hh" +#include "aterm.hh" #include #include diff --git a/src/nix-hash/Makefile.am b/src/nix-hash/Makefile.am index d7c569b68..a4fdb3246 100644 --- a/src/nix-hash/Makefile.am +++ b/src/nix-hash/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash nix_hash_SOURCES = nix-hash.cc help.txt nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-hash.o: help.txt.hh diff --git a/src/nix-instantiate/Makefile.am b/src/nix-instantiate/Makefile.am index f39331805..1c6d30873 100644 --- a/src/nix-instantiate/Makefile.am +++ b/src/nix-instantiate/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate nix_instantiate_SOURCES = nix-instantiate.cc help.txt nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-instantiate.o: help.txt.hh diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 22df2fbe5..d8b39eca2 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -11,6 +11,7 @@ #include "util.hh" #include "store-api.hh" #include "common-opts.hh" +#include "aterm.hh" #include "help.txt.hh" diff --git a/src/nix-setuid-helper/Makefile.am b/src/nix-setuid-helper/Makefile.am index a0fbdf39d..a04701636 100644 --- a/src/nix-setuid-helper/Makefile.am +++ b/src/nix-setuid-helper/Makefile.am @@ -2,7 +2,6 @@ libexec_PROGRAMS = nix-setuid-helper nix_setuid_helper_SOURCES = nix-setuid-helper.cc nix_setuid_helper_LDADD = ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} + ../boost/format/libformat.la -AM_CXXFLAGS = \ - -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil +AM_CXXFLAGS = -I$(srcdir)/.. -I$(srcdir)/../libutil diff --git a/src/nix-store/Makefile.am b/src/nix-store/Makefile.am index 0d0d0e033..e0ba809dc 100644 --- a/src/nix-store/Makefile.am +++ b/src/nix-store/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-store nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-store.o: help.txt.hh @@ -10,5 +10,5 @@ nix-store.o: help.txt.hh ../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1) AM_CXXFLAGS = \ - -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil \ + -I$(srcdir)/.. -I$(srcdir)/../libutil \ -I$(srcdir)/../libstore -I$(srcdir)/../libmain diff --git a/src/nix-worker/Makefile.am b/src/nix-worker/Makefile.am index 82bbaf1f7..b6094a2a0 100644 --- a/src/nix-worker/Makefile.am +++ b/src/nix-worker/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker nix_worker_SOURCES = nix-worker.cc help.txt nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-worker.o: help.txt.hh @@ -10,5 +10,5 @@ nix-worker.o: help.txt.hh ../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1) AM_CXXFLAGS = \ - -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil \ + -I$(srcdir)/.. -I$(srcdir)/../libutil \ -I$(srcdir)/../libstore -I$(srcdir)/../libmain From 5414b3b2dbb539a5d1f3cd48c59178ff1bb3aa08 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 16:40:40 +0000 Subject: [PATCH 042/128] * Remove the --enable-static-nix flag. --- configure.ac | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/configure.ac b/configure.ac index 35562882c..4346dde5b 100644 --- a/configure.ac +++ b/configure.ac @@ -50,38 +50,24 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')]) test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -# Whether to produce a statically linked binary. On Cygwin, this is -# the default: dynamically linking against the ATerm DLL does work, -# except that it requires the ATerm "lib" directory to be in $PATH, as -# Windows doesn't have anything like an RPATH embedded in executable. -# Since this is kind of annoying, we use static libraries for now. - -AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix], - [produce statically linked binaries]), - static_nix=$enableval, static_nix=no) - -if test "$sys_name" = cygwin; then - static_nix=yes -fi - -if test "$static_nix" = yes; then +# Windows-specific stuff. On Cygwin, dynamically linking against the +# ATerm DLL works, except that it requires the ATerm "lib" directory +# to be in $PATH, as Windows doesn't have anything like an RPATH +# embedded in executable. Since this is kind of annoying, we use +# static libraries for now. +if test "$sys_name" = "cygwin"; then AC_DISABLE_SHARED AC_ENABLE_STATIC fi -# Windows-specific stuff. -if test "$sys_name" = "cygwin"; then - # We cannot delete open files. - AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.]) -fi - # Solaris-specific stuff. if test "$sys_name" = "sunos"; then # Solaris requires -lsocket -lnsl for network functions LIBS="-lsocket -lnsl $LIBS" fi + AC_PROG_CC AC_PROG_CXX @@ -291,14 +277,6 @@ if test "$(uname)" = "Darwin"; then fi -if test "$static_nix" = yes; then - # `-all-static' has to be added at the end of configure, because - # the C compiler doesn't know about -all-static (it's filtered out - # by libtool, but configure doesn't use libtool). - LDFLAGS="-all-static $LDFLAGS" -fi - - AM_CONFIG_HEADER([config.h]) AC_CONFIG_FILES([Makefile externals/Makefile From e07d7284a221a11ceda340dea80ed392e0a8a381 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 16:55:07 +0000 Subject: [PATCH 043/128] * Fix DESTDIR builds. --- externals/Makefile.am | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/externals/Makefile.am b/externals/Makefile.am index 06ee9d590..b32256917 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -18,15 +18,15 @@ if HAVE_ATERM build-aterm: else build-aterm: $(ATERM) - (pfx=`pwd` && \ - cd $(ATERM) && \ - CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm --libdir=${pkglibdir} && \ + (cd $(ATERM) && \ + CC="$(CC)" ./configure --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} && \ $(MAKE) && \ $(MAKE) check) touch build-aterm install-exec-local:: cd $(ATERM) && make install + rm -rf "$(DESTDIR)/$(pkglibdir)/dummy" endif @@ -48,10 +48,9 @@ if HAVE_BZIP2 build-bzip2: else build-bzip2: $(BZIP2) - (pfx=`pwd` && \ - cd $(BZIP2) && \ + (cd $(BZIP2) && \ $(MAKE) && \ - $(MAKE) install PREFIX=$$pfx/inst-bzip2) + $(MAKE) install PREFIX=$(abs_builddir)/inst-bzip2) touch build-bzip2 install-exec-local:: @@ -79,14 +78,14 @@ if HAVE_SQLITE build-sqlite: else build-sqlite: $(SQLITE) - (pfx=`pwd` && \ - cd $(SQLITE) && \ - CC="$(CC)" ./configure --disable-static --prefix=$$pfx/inst-sqlite --libdir=${pkglibdir} && \ + (cd $(SQLITE) && \ + CC="$(CC)" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} && \ $(MAKE) ) touch build-sqlite install-exec-local:: cd $(SQLITE) && make install + rm -rf "$(DESTDIR)/$(pkglibdir)/dummy" endif @@ -97,4 +96,4 @@ EXTRA_DIST = $(ATERM).tar.gz $(BZIP2).tar.gz $(SQLITE_TAR) max-long.patch sizeof clean: $(RM) -f build-aterm build-bzip2 build-sqlite $(RM) -rf $(ATERM) $(BZIP2) $(SQLITE) - $(RM) -rf inst-aterm inst-bzip2 inst-sqlite + $(RM) -rf inst-bzip2 From d8c5745c41de1a1217cfeac2d4af85146cb87e0f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 17:12:47 +0000 Subject: [PATCH 044/128] * Support static builds. --- configure.ac | 7 +++++++ externals/Makefile.am | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 4346dde5b..2c4567c8d 100644 --- a/configure.ac +++ b/configure.ac @@ -77,6 +77,13 @@ AC_DISABLE_STATIC AC_ENABLE_SHARED AC_PROG_LIBTOOL +if test "$enable_shared" = yes; then + SUB_CONFIGURE_FLAGS="--enable-shared --disable-static" +else + SUB_CONFIGURE_FLAGS="--enable-static --disable-shared" +fi +AC_SUBST(SUB_CONFIGURE_FLAGS) + # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE diff --git a/externals/Makefile.am b/externals/Makefile.am index b32256917..3fc13314f 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -19,7 +19,7 @@ build-aterm: else build-aterm: $(ATERM) (cd $(ATERM) && \ - CC="$(CC)" ./configure --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} && \ + CC="$(CC)" ./configure --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \ $(MAKE) && \ $(MAKE) check) touch build-aterm @@ -79,7 +79,7 @@ build-sqlite: else build-sqlite: $(SQLITE) (cd $(SQLITE) && \ - CC="$(CC)" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} && \ + CC="$(CC)" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \ $(MAKE) ) touch build-sqlite From 3f9e647ae8c4928a8fc4de0d704119245f58ff45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 19:04:17 +0000 Subject: [PATCH 045/128] * checkInterrupt() shouldn't be called from a destructor. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 95e775ce7..8a690b913 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -47,6 +47,7 @@ SQLite::~SQLite() void SQLiteStmt::create(sqlite3 * db, const string & s) { + checkInterrupt(); assert(!stmt); if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) throw SQLiteError(db, "creating statement"); @@ -56,7 +57,6 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) void SQLiteStmt::reset() { - checkInterrupt(); assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); From 56af8e86e3e00d8417838f582221e60e1d5f5fdb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 20:09:12 +0000 Subject: [PATCH 046/128] * Protect the `true', `false' and `__overrides' constants. Without an ATprotect call, these could be garbage collected, leading to weird crashes or wrong results. --- src/aterm-helper.pl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aterm-helper.pl b/src/aterm-helper.pl index 9b2bde700..f1eb77ee8 100755 --- a/src/aterm-helper.pl +++ b/src/aterm-helper.pl @@ -146,8 +146,9 @@ while () { my $value = $2; print HEADER "extern ATerm $name;\n"; print IMPL "ATerm $name = 0;\n"; - $init .= " $name = $value;\n"; - } + $init .= " $name = $value;\n"; + $init .= " ATprotect(&$name);\n"; + } elsif (/^\s*init\s+(\w+)\s*$/) { $initFun = $1; From 158aa8931776c61e19cec62e7cea7c45961fdcc7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 20:22:46 +0000 Subject: [PATCH 047/128] * Ensure that `make install' works without a prior `make'. --- externals/Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/externals/Makefile.am b/externals/Makefile.am index 3fc13314f..2fde866ea 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -24,7 +24,7 @@ build-aterm: $(ATERM) $(MAKE) check) touch build-aterm -install-exec-local:: +install-exec-local:: build-aterm cd $(ATERM) && make install rm -rf "$(DESTDIR)/$(pkglibdir)/dummy" endif @@ -53,7 +53,7 @@ build-bzip2: $(BZIP2) $(MAKE) install PREFIX=$(abs_builddir)/inst-bzip2) touch build-bzip2 -install-exec-local:: +install-exec-local:: build-bzip2 mkdir -p $(DESTDIR)${bzip2_bin} $(INSTALL_PROGRAM) $(bzip2_bin_test)/bzip2 $(bzip2_bin_test)/bunzip2 $(DESTDIR)${bzip2_bin} endif @@ -83,7 +83,7 @@ build-sqlite: $(SQLITE) $(MAKE) ) touch build-sqlite -install-exec-local:: +install-exec-local:: build-sqlite cd $(SQLITE) && make install rm -rf "$(DESTDIR)/$(pkglibdir)/dummy" endif From e14e2399ed5b1ffc30f08d1f30f19d2ceb24dabb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 20:23:42 +0000 Subject: [PATCH 048/128] * Prevent a potential memory corruption problem if an ATerm garbage collection happens during fixAttrs(). --- src/libexpr/parser.y | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index c4afb72ea..8706ce025 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -85,6 +85,10 @@ static Expr fixAttrs(bool recursive, ATermList as) { Tree attrs; + /* This ATermMap is needed to ensure that the `leaf' fields in the + Tree nodes are not garbage collected. */ + ATermMap gcRoots; + for (ATermIterator i(as); i; ++i) { ATermList names, attrPath; Expr src, e; ATerm name, pos; @@ -95,7 +99,9 @@ static Expr fixAttrs(bool recursive, ATermList as) throw ParseError(format("duplicate definition of attribute `%1%' at %2%") % showAttrPath(ATmakeList1(*j)) % showPos(pos)); Tree & t(attrs.children[*j]); - t.leaf = fromScope ? makeVar(*j) : makeSelect(src, *j); + Expr leaf = fromScope ? makeVar(*j) : makeSelect(src, *j); + gcRoots.set(leaf, leaf); + t.leaf = leaf; t.pos = pos; if (recursive && fromScope) t.recursive = false; } From 5ff87c982e913d14adeba1c4ee833738a6b58293 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Mar 2010 16:17:09 +0000 Subject: [PATCH 049/128] From bc6f7fc139b5a72306a54c89db74bf126cca9ca7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Mar 2010 13:03:26 +0000 Subject: [PATCH 050/128] * Remove some unused functions. --- src/libstore/store-api.hh | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b6a8ff40e..095fdd24b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -112,29 +112,11 @@ public: virtual void queryReferences(const Path & path, PathSet & references) = 0; - /* Like queryReferences, but with self-references filtered out. */ - PathSet queryReferencesNoSelf(const Path & path) - { - PathSet res; - queryReferences(path, res); - res.erase(path); - return res; - } - /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const Path & path, PathSet & referrers) = 0; - /* Like queryReferrers, but with self-references filtered out. */ - PathSet queryReferrersNoSelf(const Path & path) - { - PathSet res; - queryReferrers(path, res); - res.erase(path); - return res; - } - /* Query the deriver of a store path. Return the empty string if no deriver has been set. */ virtual Path queryDeriver(const Path & path) = 0; From 04791840f4dd4d6bcc96aea133e9fda7c03897de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Mar 2010 12:54:58 +0000 Subject: [PATCH 051/128] * Emit warning='1' or error='1' attributes for lines marked as warnings or errors with \e[w or \e[e. --- src/nix-log2xml/log2xml.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/nix-log2xml/log2xml.cc b/src/nix-log2xml/log2xml.cc index b2a25eefa..6645dc500 100644 --- a/src/nix-log2xml/log2xml.cc +++ b/src/nix-log2xml/log2xml.cc @@ -18,6 +18,8 @@ struct Decoder int priority; bool ignoreLF; int lineNo, charNo; + bool warning; + bool error; Decoder() { @@ -29,6 +31,8 @@ struct Decoder ignoreLF = false; lineNo = 1; charNo = 0; + warning = false; + error = false; } void pushChar(char c); @@ -95,6 +99,12 @@ void Decoder::pushChar(char c) case 'b': ignoreLF = false; break; + case 'e': + error = true; + break; + case 'w': + warning = true; + break; } } else if (c >= '0' && c <= '9') { int n = 0; @@ -118,6 +128,8 @@ void Decoder::finishLine() string tag = inHeader ? "head" : "line"; cout << "<" << tag; if (priority != 1) cout << " priority='" << priority << "'"; + if (warning) cout << " warning='1'"; + if (error) cout << " error='1'"; cout << ">"; for (unsigned int i = 0; i < line.size(); i++) { @@ -158,6 +170,8 @@ void Decoder::finishLine() line = ""; inHeader = false; priority = 1; + warning = false; + error = false; } From 2e4ef03aa3247782339f3d5af8547b448d38b8d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Mar 2010 10:35:45 +0000 Subject: [PATCH 052/128] * Increase the sqlite timeout. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8a690b913..c7232056f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -282,7 +282,7 @@ void LocalStore::openDB(bool create) SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error("cannot open SQLite database"); - if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) throw SQLiteError(db, "setting timeout"); if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) From 44f6e6de77dd318800775d594b1f33cffa2be9a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Mar 2010 21:31:42 +0000 Subject: [PATCH 053/128] * Set `gc-keep-outputs' or `gc-keep-derivations' to false with `--delete --ignore-liveness'. --- src/libstore/gc.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index cf073c5d9..87e0a05bc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -563,6 +563,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false); state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true); + + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `gc-keep-outputs' or `gc-keep-derivations' are + true (the garbage collector will recurse into deleting the + outputs or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + state.gcKeepOutputs = false; + state.gcKeepDerivations = false; + } /* Acquire the global GC root. This prevents a) New roots from being added. From 4c356acd044dffbf459ac895b483b49959042931 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Mar 2010 14:32:03 +0000 Subject: [PATCH 054/128] * In `nix-store --export', abort if the contents of a path has changed. This prevents corrupt paths from spreading to other machines. Note that checking the hash is cheap because we're hashing anyway (because of the --sign feature). --- src/libstore/local-store.cc | 23 +++++++++++++++++------ src/libutil/hash.cc | 7 +++++++ src/libutil/hash.hh | 3 ++- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c7232056f..ff7196644 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -930,16 +930,19 @@ struct HashAndWriteSink : Sink { Sink & writeSink; HashSink hashSink; - bool hashing; HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) { - hashing = true; } virtual void operator () (const unsigned char * data, unsigned int len) { writeSink(data, len); - if (hashing) hashSink(data, len); + hashSink(data, len); + } + Hash currentHash() + { + HashSink hashSinkClone(hashSink); + return hashSinkClone.finish(); } }; @@ -970,6 +973,15 @@ void LocalStore::exportPath(const Path & path, bool sign, dumpPath(path, hashAndWriteSink); + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. */ + Hash hash = hashAndWriteSink.currentHash(); + Hash storedHash = queryPathHash(path); + if (hash != storedHash) + throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path + % printHash(storedHash) % printHash(hash)); + printMsg(lvlError, printHash(hash)); + writeInt(EXPORT_MAGIC, hashAndWriteSink); writeString(path, hashAndWriteSink); @@ -982,9 +994,8 @@ void LocalStore::exportPath(const Path & path, bool sign, writeString(deriver, hashAndWriteSink); if (sign) { - Hash hash = hashAndWriteSink.hashSink.finish(); - hashAndWriteSink.hashing = false; - + Hash hash = hashAndWriteSink.currentHash(); + writeInt(1, hashAndWriteSink); Path tmpDir = createTempDir(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index eef01fe4d..bd7e33a48 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -289,6 +289,13 @@ HashSink::HashSink(HashType ht) : ht(ht) start(ht, *ctx); } +HashSink::HashSink(const HashSink & h) +{ + ht = h.ht; + ctx = new Ctx; + *ctx = *h.ctx; +} + HashSink::~HashSink() { delete ctx; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 062d97254..81425b234 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -96,6 +96,7 @@ private: public: HashSink(HashType ht); + HashSink(const HashSink & h); ~HashSink(); virtual void operator () (const unsigned char * data, unsigned int len); Hash finish(); @@ -104,5 +105,5 @@ public: } - + #endif /* !__HASH_H */ From 1a65142ec40c92b2df9973a1fc8d948e146e05d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Mar 2010 12:46:25 +0000 Subject: [PATCH 055/128] * Remove a debug statement. --- src/libstore/local-store.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ff7196644..4d82547c6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -980,7 +980,6 @@ void LocalStore::exportPath(const Path & path, bool sign, if (hash != storedHash) throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path % printHash(storedHash) % printHash(hash)); - printMsg(lvlError, printHash(hash)); writeInt(EXPORT_MAGIC, hashAndWriteSink); From 03afc34805c0dbd5f354b823cb8391fed99c6539 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Mar 2010 13:07:37 +0000 Subject: [PATCH 056/128] * Simplify. --- release.nix | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/release.nix b/release.nix index 0de1b7bb2..0ca838a07 100644 --- a/release.nix +++ b/release.nix @@ -1,4 +1,7 @@ -{ nixpkgs ? ../nixpkgs }: +{ nix ? {outPath = ./.; rev = 1234;} +, nixpkgs ? ../nixpkgs +, officialRelease ? false +}: let @@ -6,10 +9,6 @@ let tarball = - { nix ? {outPath = ./.; rev = 1234;} - , officialRelease ? false - }: - with import nixpkgs {}; releaseTools.sourceTarball { @@ -66,9 +65,7 @@ let build = - { tarball ? jobs.tarball {} - , system ? "i686-linux" - }: + { system ? "i686-linux" }: with import nixpkgs {inherit system;}; @@ -76,7 +73,7 @@ let name = "nix"; src = tarball; - buildInputs = [curl perl bzip2 openssl]; + buildInputs = [ curl perl bzip2 openssl ]; configureFlags = '' --disable-init-state @@ -86,27 +83,24 @@ let coverage = - { tarball ? jobs.tarball {} - }: - with import nixpkgs {}; releaseTools.coverageAnalysis { name = "nix-build"; src = tarball; - buildInputs = [ - curl perl bzip2 openssl - # These are for "make check" only: - graphviz libxml2 libxslt - ]; + buildInputs = + [ curl perl bzip2 openssl + # These are for "make check" only: + graphviz libxml2 libxslt + ]; configureFlags = '' --disable-init-state --disable-shared --with-aterm=${aterm} --with-bzip2=${bzip2} --with-sqlite=${sqlite} ''; - lcovFilter = ["*/boost/*" "*-tab.*"]; + lcovFilter = [ "*/boost/*" "*-tab.*" ]; # We call `dot', and even though we just use it to # syntax-check generated dot files, it still requires some @@ -151,17 +145,15 @@ let makeRPM = system: diskImageFun: prio: - { tarball ? jobs.tarball {} - }: with import nixpkgs {inherit system;}; releaseTools.rpmBuild rec { name = "nix-rpm-${diskImage.name}"; - src = tarball; + src = jobs.tarball; diskImage = diskImageFun vmTools.diskImages; memSize = 1024; - meta = { schedulingPriority = toString prio; }; + meta.schedulingPriority = prio; }; @@ -170,17 +162,15 @@ let makeDeb = system: diskImageFun: prio: - { tarball ? jobs.tarball {} - }: with import nixpkgs {inherit system;}; releaseTools.debBuild { name = "nix-deb"; - src = tarball; + src = jobs.tarball; diskImage = diskImageFun vmTools.diskImages; memSize = 1024; - meta = { schedulingPriority = toString prio; }; + meta.schedulingPriority = prio; configureFlags = "--sysconfdir=/etc"; debRequires = ["curl"]; }; From f71ea9c911294ec72f52137175ad088a2f50ab97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Mar 2010 10:38:59 +0000 Subject: [PATCH 057/128] * Use SQLite 3.6.23. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 904957881..bc024c69c 100644 --- a/configure.ac +++ b/configure.ac @@ -244,7 +244,7 @@ AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], [prefix of SQLite]), sqlite=$withval, sqlite=) AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") -SQLITE_VERSION=3.6.22 +SQLITE_VERSION=3.6.23 AC_SUBST(SQLITE_VERSION) if test -z "$sqlite"; then sqlite_lib='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)/libsqlite3.la' From 2398af13c53217b5de5821bac0f0c44e9081c23d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Apr 2010 12:43:42 +0000 Subject: [PATCH 058/128] * Add an command `nix-store --query-failed-paths' to list the cached failed paths (when using the `build-cache-failure' option). --- src/libstore/local-store.cc | 21 +++++++++++++++++++++ src/libstore/local-store.hh | 4 ++++ src/nix-store/nix-store.cc | 17 ++++++++++++++--- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4d82547c6..f93ba3639 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -327,6 +327,8 @@ void LocalStore::openDB(bool create) "insert into FailedPaths (path, time) values (?, ?);"); stmtHasPathFailed.create(db, "select time from FailedPaths where path = ?;"); + stmtQueryFailedPaths.create(db, + "select path from FailedPaths;"); stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); stmtQueryValidDerivers.create(db, @@ -508,6 +510,25 @@ bool LocalStore::hasPathFailed(const Path & path) } +PathSet LocalStore::queryFailedPaths() +{ + SQLiteStmtUse use(stmtQueryFailedPaths); + + PathSet res; + int r; + while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, "error querying failed paths"); + + return res; +} + + Hash parseHashField(const Path & path, const string & s) { string::size_type colon = s.find(':'); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0c5f04158..70fc64fdc 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -184,6 +184,9 @@ public: /* Query whether `path' previously failed to build. */ bool hasPathFailed(const Path & path); + /* Return the set of paths that have failed to build.*/ + PathSet queryFailedPaths(); + private: Path schemaPath; @@ -203,6 +206,7 @@ private: SQLiteStmt stmtInvalidatePath; SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; + SQLiteStmt stmtQueryFailedPaths; SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ddf2062c2..34bbbcd9f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -33,7 +33,7 @@ static bool indirectRoot = false; LocalStore & ensureLocalStore() { LocalStore * store2(dynamic_cast(store.get())); - if (!store2) throw Error("you don't have sufficient rights to use --verify"); + if (!store2) throw Error("you don't have sufficient rights to use this command"); return *store2; } @@ -651,8 +651,7 @@ static void opOptimise(Strings opFlags, Strings opArgs) bool dryRun = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) + foreach (Strings::iterator, i, opFlags) if (*i == "--dry-run") dryRun = true; else throw UsageError(format("unknown flag `%1%'") % *i); @@ -667,6 +666,16 @@ static void opOptimise(Strings opFlags, Strings opArgs) } +static void opQueryFailedPaths(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty() || !opFlags.empty()) + throw UsageError("no arguments expected"); + PathSet failed = ensureLocalStore().queryFailedPaths(); + foreach (PathSet::iterator, i, failed) + cout << format("%1%\n") % *i; +} + + /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ @@ -718,6 +727,8 @@ void run(Strings args) op = opVerify; else if (arg == "--optimise") op = opOptimise; + else if (arg == "--query-failed-paths") + op = opQueryFailedPaths; else if (arg == "--add-root") { if (i == args.end()) throw UsageError("`--add-root requires an argument"); From 6199f9b93ee234139906792c8c1b4908a35df146 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Apr 2010 12:56:42 +0000 Subject: [PATCH 059/128] * Added a command `nix-store --clear-failed-paths ' to clear the "failed" status of the given store paths. The special value `*' clears all failed paths. --- src/libstore/local-store.cc | 17 +++++++++++++++++ src/libstore/local-store.hh | 5 +++++ src/nix-store/nix-store.cc | 12 +++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f93ba3639..633b3599e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -329,6 +329,8 @@ void LocalStore::openDB(bool create) "select time from FailedPaths where path = ?;"); stmtQueryFailedPaths.create(db, "select path from FailedPaths;"); + stmtClearFailedPath.create(db, + "delete from FailedPaths where ?1 = '*' or path = ?1;"); stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); stmtQueryValidDerivers.create(db, @@ -529,6 +531,21 @@ PathSet LocalStore::queryFailedPaths() } +void LocalStore::clearFailedPaths(const PathSet & paths) +{ + SQLiteTxn txn(db); + + foreach (PathSet::const_iterator, i, paths) { + SQLiteStmtUse use(stmtClearFailedPath); + stmtClearFailedPath.bind(*i); + if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) + throw SQLiteError(db, format("clearing failed path `%1%' in database") % *i); + } + + txn.commit(); +} + + Hash parseHashField(const Path & path, const string & s) { string::size_type colon = s.find(':'); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 70fc64fdc..2fd640e39 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -187,6 +187,10 @@ public: /* Return the set of paths that have failed to build.*/ PathSet queryFailedPaths(); + /* Clear the "failed" status of the given paths. The special + value `*' causes all failed paths to be cleared. */ + void clearFailedPaths(const PathSet & paths); + private: Path schemaPath; @@ -207,6 +211,7 @@ private: SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; SQLiteStmt stmtQueryFailedPaths; + SQLiteStmt stmtClearFailedPath; SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 34bbbcd9f..148fd6add 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -666,7 +666,7 @@ static void opOptimise(Strings opFlags, Strings opArgs) } -static void opQueryFailedPaths(Strings opFlags, Strings opArgs) +static void opQueryFailedPaths(Strings opFlags, Strings opArgs) { if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments expected"); @@ -676,6 +676,14 @@ static void opQueryFailedPaths(Strings opFlags, Strings opArgs) } +static void opClearFailedPaths(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) + throw UsageError("no flags expected"); + ensureLocalStore().clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); +} + + /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ @@ -729,6 +737,8 @@ void run(Strings args) op = opOptimise; else if (arg == "--query-failed-paths") op = opQueryFailedPaths; + else if (arg == "--clear-failed-paths") + op = opClearFailedPaths; else if (arg == "--add-root") { if (i == args.end()) throw UsageError("`--add-root requires an argument"); From ef337f7089e484929be92114dac5455d00cebb45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Apr 2010 12:58:12 +0000 Subject: [PATCH 060/128] --- src/nix-store/help.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index 7576e3ef5..8022bf7c7 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -27,6 +27,9 @@ Operations: --verify: verify Nix structures --optimise: optimise the Nix store by hard-linking identical files + --query-failed-paths: list paths that failed to build (if enabled) + --clear-failed-paths: clear the failed status of the given paths + --version: output version information --help: display help From c778ed17687a506c46c107a7adb1f3173d8136da Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Apr 2010 13:39:55 +0000 Subject: [PATCH 061/128] * Fix the copy-from-other-stores substituter. --- scripts/copy-from-other-stores.pl.in | 56 ++++++++++++---------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/scripts/copy-from-other-stores.pl.in b/scripts/copy-from-other-stores.pl.in index 8f0ff4ca8..a6a14c3dc 100644 --- a/scripts/copy-from-other-stores.pl.in +++ b/scripts/copy-from-other-stores.pl.in @@ -17,25 +17,19 @@ foreach my $dir (@remoteStoresAll) { } +$ENV{"NIX_REMOTE"} = ""; + + sub findStorePath { my $storePath = shift; - - my $storePathName = basename $storePath; - foreach my $store (@remoteStores) { - # Determine whether $storePath exists by looking for the - # existence of the info file, and if so, get store path info - # from that file. This rather breaks abstraction: we should - # be using `nix-store' for that. But right now there is no - # good way to tell nix-store to access a store mounted under a - # different location (there's $NIX_STORE, but that only works - # if the remote store is mounted under its "real" location). - my $infoFile = "$store/var/nix/db/info/$storePathName"; - my $storePath2 = "$store/store/$storePathName"; - if (-f $infoFile && -e $storePath2) { - return ($infoFile, $storePath2); - } + my $sourcePath = "$store/store/" . basename $storePath; + next unless -e $sourcePath || -l $sourcePath; + $ENV{"NIX_DB_DIR"} = "$store/var/nix/db"; + return ($store, $sourcePath) if + system("@bindir@/nix-store --check-validity $storePath") == 0; } + return undef; } @@ -46,32 +40,28 @@ if ($ARGV[0] eq "--query") { if ($cmd eq "have") { my $storePath = ; chomp $storePath; - (my $infoFile) = findStorePath $storePath; - print STDOUT ($infoFile ? "1\n" : "0\n"); + print STDOUT (defined findStorePath($storePath) ? "1\n" : "0\n"); } elsif ($cmd eq "info") { my $storePath = ; chomp $storePath; - (my $infoFile) = findStorePath $storePath; - if (!$infoFile) { + my ($store, $sourcePath) = findStorePath($storePath); + if (!defined $store) { print "0\n"; next; # not an error } print "1\n"; - my $deriver = ""; - my @references = (); + $ENV{"NIX_DB_DIR"} = "$store/var/nix/db"; + + my $deriver = `@bindir@/nix-store --query --deriver $storePath`; + die "cannot query deriver of `$storePath'" if $? != 0; + chomp $deriver; + $deriver = "" if $deriver eq "unknown-deriver"; - open INFO, "<$infoFile" or die "cannot read info file $infoFile\n"; - while () { - chomp; - /^([\w-]+): (.*)$/ or die "bad info file"; - my $key = $1; - my $value = $2; - if ($key eq "Deriver") { $deriver = $value; } - elsif ($key eq "References") { @references = split ' ', $value; } - } - close INFO; + my @references = split "\n", + `@bindir@/nix-store --query --references $storePath`; + die "cannot query references of `$storePath'" if $? != 0; print "$deriver\n"; print scalar @references, "\n"; @@ -87,8 +77,8 @@ if ($ARGV[0] eq "--query") { elsif ($ARGV[0] eq "--substitute") { die unless scalar @ARGV == 2; my $storePath = $ARGV[1]; - (my $infoFile, my $sourcePath) = findStorePath $storePath; - die unless $infoFile; + my ($store, $sourcePath) = findStorePath $storePath; + die unless $store; print "\n*** Copying `$storePath' from `$sourcePath'\n\n"; system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0 or die "cannot copy `$sourcePath' to `$storePath'"; From 7fa338f4bac16f83b65b95fb0397b534d5ba5d5b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 2 May 2010 21:27:50 +0000 Subject: [PATCH 062/128] * Don't use smart quotes where normal ASCII quotes are intended. Actually, don't use quotes at all. (Reported by Howard B. Golden.) --- doc/manual/quick-start.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/quick-start.xml b/doc/manual/quick-start.xml index 6d96cb5ac..d2431151f 100644 --- a/doc/manual/quick-start.xml +++ b/doc/manual/quick-start.xml @@ -60,7 +60,7 @@ available remotely. in the channel: -$ nix-env -qa ’*’ (mind the quotes!) +$ nix-env -qa \* docbook-xml-4.2 firefox-1.0pre-PR-0.10.1 hello-2.1.1 From f92c9a0ac585d30e245c6667cbce4b035659cb11 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 May 2010 10:45:10 +0000 Subject: [PATCH 063/128] * Allow unprivileged users to do `nix-store --clear-failed-paths' and `nix-store --query-failed-paths'. --- src/libstore/local-store.hh | 3 --- src/libstore/remote-store.cc | 19 +++++++++++++++++++ src/libstore/remote-store.hh | 4 ++++ src/libstore/store-api.hh | 7 +++++++ src/libstore/worker-protocol.hh | 2 ++ src/nix-store/nix-store.cc | 4 ++-- src/nix-worker/nix-worker.cc | 17 +++++++++++++++++ 7 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 2fd640e39..c1e0e335f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -184,11 +184,8 @@ public: /* Query whether `path' previously failed to build. */ bool hasPathFailed(const Path & path); - /* Return the set of paths that have failed to build.*/ PathSet queryFailedPaths(); - /* Clear the "failed" status of the given paths. The special - value `*' causes all failed paths to be cleared. */ void clearFailedPaths(const PathSet & paths); private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c5d7975b5..334ad95cf 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -451,6 +451,25 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) } +PathSet RemoteStore::queryFailedPaths() +{ + openConnection(); + writeInt(wopQueryFailedPaths, to); + processStderr(); + return readStorePaths(from); +} + + +void RemoteStore::clearFailedPaths(const PathSet & paths) +{ + openConnection(); + writeInt(wopClearFailedPaths, to); + writeStringSet(paths, to); + processStderr(); + readInt(from); +} + + void RemoteStore::processStderr(Sink * sink, Source * source) { unsigned int msg; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8bab1d8c4..02a1c4752 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -70,6 +70,10 @@ public: void collectGarbage(const GCOptions & options, GCResults & results); + PathSet queryFailedPaths(); + + void clearFailedPaths(const PathSet & paths); + private: AutoCloseFD fdSocket; FdSink to; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 095fdd24b..fbe0cce81 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -209,6 +209,13 @@ public: /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /* Return the set of paths that have failed to build.*/ + virtual PathSet queryFailedPaths() = 0; + + /* Clear the "failed" status of the given paths. The special + value `*' causes all failed paths to be cleared. */ + virtual void clearFailedPaths(const PathSet & paths) = 0; }; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c3096010b..392a69acf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -36,6 +36,8 @@ typedef enum { wopQuerySubstitutablePathInfo = 21, wopQueryDerivationOutputs = 22, wopQueryValidPaths = 23, + wopQueryFailedPaths = 24, + wopClearFailedPaths = 25, } WorkerOp; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 148fd6add..3b34b9dae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -670,7 +670,7 @@ static void opQueryFailedPaths(Strings opFlags, Strings opArgs) { if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments expected"); - PathSet failed = ensureLocalStore().queryFailedPaths(); + PathSet failed = store->queryFailedPaths(); foreach (PathSet::iterator, i, failed) cout << format("%1%\n") % *i; } @@ -680,7 +680,7 @@ static void opClearFailedPaths(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("no flags expected"); - ensureLocalStore().clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); + store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index a41fb2e15..d41877e88 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -528,6 +528,23 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryFailedPaths: { + startWork(); + PathSet paths = store->queryFailedPaths(); + stopWork(); + writeStringSet(paths, to); + break; + } + + case wopClearFailedPaths: { + PathSet paths = readStringSet(from); + startWork(); + store->clearFailedPaths(paths); + stopWork(); + writeInt(1, to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } From a0e3b84fac56cad6377ecd1462058a6b29bb1ea8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 May 2010 12:42:58 +0000 Subject: [PATCH 064/128] * Revert r15436. This was a workaround for a bug in btrfs which seems to have been fixed now. --- src/libutil/archive.cc | 8 -------- src/libutil/archive.hh | 1 - 2 files changed, 9 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 8fde4328c..999b17cd2 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -181,8 +181,6 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) left -= n; } - sink.finalizeContents(size); - readPadding(size, source); } @@ -317,12 +315,6 @@ struct RestoreSink : ParseSink writeFull(fd, data, len); } - void finalizeContents(unsigned long long size) - { - errno = ftruncate(fd, size); - if (errno) throw SysError(format("truncating file to its allocated length of %1% bytes") % size); - } - void createSymlink(const Path & path, const string & target) { Path p = dstPath + path; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index f358a2a6b..fff620313 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -64,7 +64,6 @@ struct ParseSink virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; virtual void receiveContents(unsigned char * data, unsigned int len) { }; - virtual void finalizeContents(unsigned long long size) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; From a443c7573b3d76a6db107e6de974205e605a2738 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Jun 2010 10:01:14 +0000 Subject: [PATCH 065/128] * Hack to allow derivations to disable chroot builds by setting the attribute "__noChroot = true" (requested by Rob). --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e068ab0bd..a482ace00 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1545,6 +1545,9 @@ void DerivationGoal::startBuilder() if (fixedOutput) useChroot = false; + /* Hack to allow derivations to disable chroot builds. */ + if (drv.env["__noChroot"] == "1") useChroot = false; + if (useChroot) { #if CHROOT_ENABLED /* Create a temporary directory in which we set up the chroot From d1f6c0cbe39b509545f809f08cbd580859f38e34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Jun 2010 13:38:28 +0000 Subject: [PATCH 066/128] * Replacing ValidPath rows doesn't work because it causes a constraint violation of the Refs table. So don't do that. --- src/libstore/local-store.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 071f8a223..cf0d559bd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -311,7 +311,7 @@ void LocalStore::openDB(bool create) /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, - "insert or replace into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -837,7 +837,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) { SQLiteTxn txn(db); - foreach (ValidPathInfos::const_iterator, i, infos) addValidPath(*i); + foreach (ValidPathInfos::const_iterator, i, infos) + /* !!! Maybe the registration info should be updated if the + path is already valid. */ + if (!isValidPath(i->path)) addValidPath(*i); foreach (ValidPathInfos::const_iterator, i, infos) { unsigned long long referrer = queryValidPathId(i->path); From 3e5e0faf9cf93c01fb621774c0c3c50ce51bdd91 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Jun 2010 11:08:09 +0000 Subject: [PATCH 067/128] * Okay, putting a lock on the temporary directory used by importPath() doesn't work because the garbage collector doesn't actually look at locks. So r22253 was stupid. Use addTempRoot() instead. Also, locking the temporary directory in exportPath() was silly because it isn't even in the store. --- src/libstore/local-store.cc | 24 ++++++++++++++++++------ src/libstore/local-store.hh | 2 ++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c7530f1b1..6f3d9efa8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1036,8 +1036,6 @@ void LocalStore::exportPath(const Path & path, bool sign, writeInt(1, hashAndWriteSink); Path tmpDir = createTempDir(); - PathLocks tmpDirLock(singleton(tmpDir)); - tmpDirLock.setDeletion(true); AutoDelete delTmp(tmpDir); Path hashFile = tmpDir + "/hash"; writeFile(hashFile, printHash(hash)); @@ -1079,6 +1077,22 @@ struct HashAndReadSource : Source }; +/* Create a temporary directory in the store that won't be + garbage-collected. */ +Path LocalStore::createTempDirInStore() +{ + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(nixStore); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; +} + + Path LocalStore::importPath(bool requireSignature, Source & source) { HashAndReadSource hashAndReadSource(source); @@ -1086,10 +1100,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* We don't yet know what store path this archive contains (the store path follows the archive data proper), and besides, we don't know yet whether the signature is valid. */ - Path tmpDir = createTempDir(nixStore); - PathLocks tmpDirLock(singleton(tmpDir)); - tmpDirLock.setDeletion(true); - AutoDelete delTmp(tmpDir); /* !!! could be GC'ed! */ + Path tmpDir = createTempDirInStore(); + AutoDelete delTmp(tmpDir); Path unpacked = tmpDir + "/unpacked"; restorePath(unpacked, hashAndReadSource); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e0e335f..3ae964f05 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -246,6 +246,8 @@ private: void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); + + Path createTempDirInStore(); }; From 7f893b7a43fdca728fd1f7a72e51d31d2a6e7147 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Aug 2010 12:13:58 +0000 Subject: [PATCH 068/128] * Allow derivations to hint that they should not be built remotely using the build hook mechanism, by setting the derivation attribute "preferLocalBuild" to true. This has a few use cases: - The user environment builder. Since it just creates a bunch of symlinks without much computation, there is no reason to do it remotely. In fact, doing it remotely requires the entire closure of the user environment to be copied to the remote machine, which is extremely wasteful. - `fetchurl'. Performing the download on a remote machine and then copying it to the local machine involves twice as much network traffic as performing the download locally, and doesn't save any CPU cycles on the local machine. --- corepkgs/buildenv/default.nix | 4 +++ src/libstore/build.cc | 64 ++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/corepkgs/buildenv/default.nix b/corepkgs/buildenv/default.nix index 36dd9d0c6..d76f52740 100644 --- a/corepkgs/buildenv/default.nix +++ b/corepkgs/buildenv/default.nix @@ -11,4 +11,8 @@ derivation { paths = derivations; active = map (x: if x ? meta && x.meta ? active then x.meta.active else "true") derivations; priority = map (x: if x ? meta && x.meta ? priority then x.meta.priority else "5") derivations; + + # Building user environments remotely just causes huge amounts of + # network traffic, so don't do that. + preferLocalBuild = true; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8ce8c873d..5818aa51d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -960,6 +960,16 @@ PathSet outputPaths(const DerivationOutputs & outputs) } +static bool canBuildLocally(const string & platform) +{ + return platform == thisSystem +#ifdef CAN_DO_LINUX32_BUILDS + || (platform == "i686-linux" && thisSystem == "x86_64-linux") +#endif + ; +} + + void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1027,28 +1037,38 @@ void DerivationGoal::tryToBuild() foreach (DerivationOutputs::iterator, i, drv.outputs) if (pathFailed(i->second.path)) return; - /* Is the build hook willing to accept this job? */ - usingBuildHook = true; - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get EOF - from the hook. */ - state = &DerivationGoal::buildDone; - return; - case rpPostpone: - /* Not now; wait until at least one child finishes. */ - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. */ + bool preferLocalBuild = + drv.env["preferLocalBuild"] == "1" && canBuildLocally(drv.platform); + /* Is the build hook willing to accept this job? */ + if (!preferLocalBuild) { + usingBuildHook = true; + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + state = &DerivationGoal::buildDone; + return; + case rpPostpone: + /* Not now; wait until at least one child finishes. */ + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } + } + usingBuildHook = false; - /* Make sure that we are allowed to start a build. */ - if (worker.getNrLocalBuilds() >= maxBuildJobs) { + /* Make sure that we are allowed to start a build. If this + derivation prefers to be done locally, do it even if + maxBuildJobs is 0. */ + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; @@ -1379,11 +1399,7 @@ void DerivationGoal::startBuilder() format("building path(s) %1%") % showPaths(outputPaths(drv.outputs))) /* Right platform? */ - if (drv.platform != thisSystem -#ifdef CAN_DO_LINUX32_BUILDS - && !(drv.platform == "i686-linux" && thisSystem == "x86_64-linux") -#endif - ) + if (!canBuildLocally(drv.platform)) throw Error( format("a `%1%' is required to build `%3%', but I am a `%2%'") % drv.platform % thisSystem % drvPath); From fd9c77dfc7b90d447e6bfdb4f0d5b521184aeddb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Aug 2010 17:35:59 +0000 Subject: [PATCH 069/128] * Use SQLite 3.7.0's write-ahead logging (WAL mode). This is a lot faster than the old mode when fsyncs are enabled, because it only performs an fsync() when doing a checkpoint, rather than at every commit. Some timings for doing a "nix-instantiate /etc/nixos/nixos -A system" after modifying the stdenv setup script: 42.5s - SQLite 3.6.23 with truncate mode and fsync 3.4s - SQLite 3.6.23 with truncate mode and no fsync 32.1s - SQLite 3.7.0 with truncate mode and fsync 16.8s - SQLite 3.7.0 with WAL mode and fsync, auto-checkpoint every 1000 pages 8.3s - SQLite 3.7.0 with WAL mode and fsync, auto-checkpoint every 8192 pages 1.7s - SQLite 3.7.0 with WAL mode and no fsync The default is now to use WAL mode with fsyncs. Because WAL doesn't work on remote filesystems such as NFS (as it uses shared memory), truncate mode can be re-enabled by setting the "use-sqlite-wal" option to false. --- bootstrap.sh | 1 + configure.ac | 2 +- src/libstore/local-store.cc | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index f007c713b..2547f5dc8 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,4 +1,5 @@ #! /bin/sh -e +rm -f aclocal.m4 mkdir -p config libtoolize --copy aclocal diff --git a/configure.ac b/configure.ac index 662a65c82..acf5ac7ce 100644 --- a/configure.ac +++ b/configure.ac @@ -236,7 +236,7 @@ AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], [prefix of SQLite]), sqlite=$withval, sqlite=) AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") -SQLITE_VERSION=3.6.23 +SQLITE_VERSION=3.7.0 AC_SUBST(SQLITE_VERSION) if test -z "$sqlite"; then sqlite_lib='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)/libsqlite3.la' diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6f3d9efa8..bddcbad12 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -298,10 +298,20 @@ void LocalStore::openDB(bool create) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting synchronous mode"); - /* Use `truncate' journal mode, which should be a bit faster. */ - if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) + /* Set the SQLite journal mode. The default is write-ahead + logging since it's the fastest and supports more concurrency. + The downside is that it doesn't work over NFS, so allow + truncate mode alternatively. */ + string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; + if (sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting journal mode"); + /* Increase the auto-checkpoint interval to 8192 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting autocheckpoint interval"); + /* Initialise the database schema, if necessary. */ if (create) { #include "schema.sql.hh" From d7875d164884b1a42d9b5ed0d92b55beeeb89a99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 6 Aug 2010 07:51:27 +0000 Subject: [PATCH 070/128] * Use SQLite 3.7.0.1. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bf18abea3..867e29029 100644 --- a/configure.ac +++ b/configure.ac @@ -240,7 +240,7 @@ AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], [prefix of SQLite]), sqlite=$withval, sqlite=) AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") -SQLITE_VERSION=3.7.0 +SQLITE_VERSION=3.7.0.1 AC_SUBST(SQLITE_VERSION) if test -z "$sqlite"; then sqlite_lib='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)/libsqlite3.la' From 5f9aad44caff5b2a2fb22fcf93d6ca129656984d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Aug 2010 15:28:02 +0000 Subject: [PATCH 071/128] * nix-build: recognise --cores. --- scripts/nix-build.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index d27eb08e2..ed85d5712 100644 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -102,7 +102,7 @@ EOF $n += 2; } - elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type") { + elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores") { $n++; die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; push @buildArgs, ($arg, $ARGV[$n]); From 5c5ab2bc12472f63a33cc841dcdc57f1ed8ddea5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Aug 2010 13:34:34 +0000 Subject: [PATCH 072/128] * Don't link against pthreads. This was added way back in r211 because Berkeley DB needed it on some platforms, but we don't use BDB anymore. On FreeBSD, if you link against pthreads, then the main thread gets a 2 MB stack which cannot be overriden (it ignores "ulimit -s"): http://www.mail-archive.com/freebsd-hackers@freebsd.org/msg62445.html This is not enough for Nix. For instance, the garbage collector can fail if there is a pathologically deep chain of references (http://hydra.nixos.org/build/556199). 2 MB is also not enough for many Nix expressions. Arguably the garbage collector shouldn't use recursion, because in NixOS unprivileged users can DOS the garbage collector by creating a sufficiently deeply nested chain of references. But getting rid of recursion is a bit harder. --- configure.ac | 3 --- 1 file changed, 3 deletions(-) diff --git a/configure.ac b/configure.ac index 867e29029..0b7f89448 100644 --- a/configure.ac +++ b/configure.ac @@ -256,9 +256,6 @@ AC_SUBST(sqlite_include) AC_SUBST(sqlite_bin) -AC_CHECK_LIB(pthread, pthread_mutex_init) - - AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state], [do not initialise DB etc. in `make install']), init_state=$enableval, init_state=yes) From f58f51f38007f8bfb1089c3b4c88b6e66da15f39 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Aug 2010 11:45:44 +0000 Subject: [PATCH 073/128] * Handle the unlikely case where a derivation has no dependencies at all. --- src/libstore/build.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6f0276244..5617eccf1 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -887,7 +887,10 @@ void DerivationGoal::outputsSubstituted() foreach (PathSet::iterator, i, drv.inputSrcs) addWaitee(worker.makeSubstitutionGoal(*i)); - state = &DerivationGoal::inputsRealised; + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; } From 034f608e004c3206ff99c55c107139c38bfe9408 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Aug 2010 14:25:33 +0000 Subject: [PATCH 074/128] * Don't complain if the stored hash of a path is zero (unknown). --- src/libstore/local-store.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bddcbad12..9254ceee4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,10 +1022,11 @@ void LocalStore::exportPath(const Path & path, bool sign, dumpPath(path, hashAndWriteSink); /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. */ + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); Hash storedHash = queryPathHash(path); - if (hash != storedHash) + if (hash != storedHash && storedHash != Hash(storedHash.type)) throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path % printHash(storedHash) % printHash(hash)); From 1e5f5ea2e9f522397c05e710ae32ff7c0b0f1611 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Aug 2010 14:27:07 +0000 Subject: [PATCH 075/128] * Correctly handle SSH failing to establish a connection. Because this didn't work right, the build hook wouldn't fall back to using other machines of the desired type. --- scripts/ssh.pm | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/ssh.pm b/scripts/ssh.pm index c6d667a65..44a0e6f31 100644 --- a/scripts/ssh.pm +++ b/scripts/ssh.pm @@ -25,13 +25,16 @@ sub openSSHConnection { # print "started" when it has established the connection, and wait # until we see that. open SSH, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die; + while () { chomp; - last if /started/; + if ($_ eq "started") { + $sshStarted = 1; + return 1; + } } - - $sshStarted = 1; - return 1; + + return 0; } # Tell the master SSH client to exit. From 95deba581dc3a93874b13406fb5c4dfb6747bbc3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Aug 2010 11:54:11 +0000 Subject: [PATCH 076/128] * In the build hook, temporarily register the derivation and its output as GC roots. This prevents a race if the garbage collector is running during the build. --- scripts/build-remote.pl.in | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index c440b6a0f..2fb29dcfc 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -192,7 +192,6 @@ if ($x ne "okay") { } -# Do the actual build. print STDERR "building `$drvPath' on `$hostName'\n"; my $inputs = `cat inputs`; die if ($? != 0); @@ -201,17 +200,29 @@ $inputs =~ s/\n/ /g; my $outputs = `cat outputs`; die if ($? != 0); $outputs =~ s/\n/ /g; -print "copying inputs...\n"; - my $maybeSign = ""; $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec"; + +# Register the derivation as a temporary GC root. Note that $PPID is +# the PID of the remote SSH process, which, due to the use of a +# persistant SSH connection, should be the same across all remote +# command invocations for this session. +my $rootsDir = "@localstatedir@/nix/gcroots/tmp"; +system("ssh $hostName @sshOpts 'mkdir -m 1777 -p $rootsDir; ln -sfn $drvPath $rootsDir/\$PPID.drv'"); + +sub removeRoots { + system("ssh $hostName @sshOpts 'rm -f $rootsDir/\$PPID.drv $rootsDir/\$PPID.out'"); +} + + +# Copy the derivation and its dependencies to the build machine. system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath $inputs") == 0 or die "cannot copy inputs to $hostName: $?"; -print "building...\n"; -my $buildFlags = "--max-silent-time $maxSilentTime --fallback"; +# Perform the build. +my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsDir/\$PPID.out"; # `-tt' forces allocation of a pseudo-terminal. This is required to # make the remote nix-store process receive a signal when the @@ -226,11 +237,14 @@ if (system("ssh $hostName @sshOpts -tt 'nix-store -r $drvPath $buildFlags > /dev # the first is a transient failure and the latter is permanent. my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100; print STDERR "build of `$drvPath' on `$hostName' failed with exit code $?\n"; + removeRoots; exit $res; } print "build of `$drvPath' on `$hostName' succeeded\n"; + +# Copy the output from the build machine. foreach my $output (split '\n', $outputs) { my $maybeSignRemote = ""; $maybeSignRemote = "--sign" if $UID != 0; @@ -238,3 +252,7 @@ foreach my $output (split '\n', $outputs) { system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output' | @bindir@/nix-store --import > /dev/null") == 0 or die "cannot copy $output from $hostName: $?"; } + + +# Get rid of the temporary GC roots. +removeRoots; From 1a396f3789feefc20bdcd8a355939eb1ec5126c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Aug 2010 12:19:30 +0000 Subject: [PATCH 077/128] * Don't call "cat". --- scripts/build-remote.pl.in | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 2fb29dcfc..9b3d98cd5 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -194,11 +194,8 @@ if ($x ne "okay") { print STDERR "building `$drvPath' on `$hostName'\n"; -my $inputs = `cat inputs`; die if ($? != 0); -$inputs =~ s/\n/ /g; - -my $outputs = `cat outputs`; die if ($? != 0); -$outputs =~ s/\n/ /g; +my @inputs = split /\s/, do { local $/; local @ARGV = "inputs"; <> }; +my @outputs = split /\s/, do { local $/; local @ARGV = "outputs"; <> }; my $maybeSign = ""; $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec"; @@ -217,7 +214,7 @@ sub removeRoots { # Copy the derivation and its dependencies to the build machine. -system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath $inputs") == 0 +system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0 or die "cannot copy inputs to $hostName: $?"; @@ -245,7 +242,7 @@ print "build of `$drvPath' on `$hostName' succeeded\n"; # Copy the output from the build machine. -foreach my $output (split '\n', $outputs) { +foreach my $output (@outputs) { my $maybeSignRemote = ""; $maybeSignRemote = "--sign" if $UID != 0; From e437b0825018b1935f9a849382c12b1df0aeae06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Aug 2010 20:44:28 +0000 Subject: [PATCH 078/128] * Made the build hook mechanism more efficient. Rather than starting the hook every time we want to ask whether we can run a remote build (which can be very often), we now reuse a hook process for answering those queries until it accepts a build. So if there are N derivations to be built, at most N hooks will be started. --- scripts/build-remote.pl.in | 258 ++++++++++++------------ src/libstore/build.cc | 392 ++++++++++++++++++------------------- tests/build-hook.hook.sh | 28 +-- 3 files changed, 332 insertions(+), 346 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 9b3d98cd5..a8f73f2e5 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -31,57 +31,20 @@ $ENV{"DISPLAY"} = ""; $ENV{"SSH_ASKPASS"} = ""; -my $loadIncreased = 0; - -my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV; -$maxSilentTime = 0 unless defined $maxSilentTime; - sub sendReply { my $reply = shift; print STDERR "# $reply\n"; } -sub decline { - sendReply "decline"; - exit 0; -} + +# Initialisation. +my $loadIncreased = 0; + +my ($localSystem, $maxSilentTime) = @ARGV; +$maxSilentTime = 0 unless defined $maxSilentTime; my $currentLoad = $ENV{"NIX_CURRENT_LOAD"}; -decline unless defined $currentLoad; -mkdir $currentLoad, 0777 or die unless -d $currentLoad; - my $conf = $ENV{"NIX_REMOTE_SYSTEMS"}; -decline if !defined $conf || ! -e $conf; - -my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem); - - -# Read the list of machines. -my @machines; -open CONF, "< $conf" or die; - -while () { - chomp; - s/\#.*$//g; - next if /^\s*$/; - /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)(\s+([0-9\.]+))?\s*$/ or die; - push @machines, - { hostName => $1 - , systemTypes => [split(/,/, $2)] - , sshKeys => $3 - , maxJobs => $4 - , speedFactor => 1.0 * ($6 || 1) - , enabled => 1 - }; -} - -close CONF; - - -# Acquire the exclusive lock on $currentLoad/main-lock. -my $mainLock = "$currentLoad/main-lock"; -open MAINLOCK, ">>$mainLock" or die; -flock(MAINLOCK, LOCK_EX) or die; sub openSlotLock { @@ -91,111 +54,147 @@ sub openSlotLock { open $slotLock, ">>$slotLockFn" or die; return $slotLock; } + + +# Read the list of machines. +my @machines; +if (defined $conf && -e $conf) { + open CONF, "< $conf" or die; + while () { + chomp; + s/\#.*$//g; + next if /^\s*$/; + /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)(\s+([0-9\.]+))?\s*$/ or die; + push @machines, + { hostName => $1 + , systemTypes => [split(/,/, $2)] + , sshKeys => $3 + , maxJobs => $4 + , speedFactor => 1.0 * ($6 || 1) + , enabled => 1 + }; + } + close CONF; +} + + + +# Wait for the calling process to ask us whether we can build some derivation. +my ($drvPath, $hostName, $slotLock); + +REQ: while (1) { + $_ = || exit 0; + my ($amWilling, $neededSystem); + ($amWilling, $neededSystem, $drvPath) = split; + + my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem); + + if (!defined $currentLoad) { + sendReply "decline"; + next; + } - -my $hostName; -my $slotLock; - -while (1) { + # Acquire the exclusive lock on $currentLoad/main-lock. + mkdir $currentLoad, 0777 or die unless -d $currentLoad; + my $mainLock = "$currentLoad/main-lock"; + open MAINLOCK, ">>$mainLock" or die; + flock(MAINLOCK, LOCK_EX) or die; - # Find all machine that can execute this build, i.e., that support - # builds for the given platform and are not at their job limit. - my $rightType = 0; - my @available = (); - LOOP: foreach my $cur (@machines) { - if ($cur->{enabled} && grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) { - $rightType = 1; + + while (1) { + # Find all machine that can execute this build, i.e., that + # support builds for the given platform and are not at their + # job limit. + my $rightType = 0; + my @available = (); + LOOP: foreach my $cur (@machines) { + if ($cur->{enabled} && grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) { + $rightType = 1; - # We have a machine of the right type. Determine the load on - # the machine. - my $slot = 0; - my $load = 0; - my $free; - while ($slot < $cur->{maxJobs}) { - my $slotLock = openSlotLock($cur, $slot); - if (flock($slotLock, LOCK_EX | LOCK_NB)) { - $free = $slot unless defined $free; - flock($slotLock, LOCK_UN) or die; - } else { - $load++; + # We have a machine of the right type. Determine the load on + # the machine. + my $slot = 0; + my $load = 0; + my $free; + while ($slot < $cur->{maxJobs}) { + my $slotLock = openSlotLock($cur, $slot); + if (flock($slotLock, LOCK_EX | LOCK_NB)) { + $free = $slot unless defined $free; + flock($slotLock, LOCK_UN) or die; + } else { + $load++; + } + close $slotLock; + $slot++; } - close $slotLock; - $slot++; + + push @available, { machine => $cur, load => $load, free => $free } + if $load < $cur->{maxJobs}; } - - push @available, { machine => $cur, load => $load, free => $free } - if $load < $cur->{maxJobs}; } - } - if (defined $ENV{NIX_DEBUG_HOOK}) { - print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n" - foreach @available; - } - - - # Didn't find any available machine? Then decline or postpone. - if (scalar @available == 0) { - # Postpone if we have a machine of the right type, except if the - # local system can and wants to do the build. - if ($rightType && !$canBuildLocally) { - sendReply "postpone"; - exit 0; - } else { - decline; + if (defined $ENV{NIX_DEBUG_HOOK}) { + print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n" + foreach @available; } - } - # Prioritise the available machines as follows: - # - First by load divided by speed factor, rounded to the nearest - # integer. This causes fast machines to be preferred over slow - # machines with similar loads. - # - Then by speed factor. - # - Finally by load. - sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); } - @available = sort - { lf($a) <=> lf($b) - || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor} - || $a->{load} <=> $b->{load} - } @available; + # Didn't find any available machine? Then decline or postpone. + if (scalar @available == 0) { + # Postpone if we have a machine of the right type, except + # if the local system can and wants to do the build. + if ($rightType && !$canBuildLocally) { + sendReply "postpone"; + } else { + sendReply "decline"; + } + close MAINLOCK; + next REQ; + } - # Select the best available machine and lock a free slot. - my $selected = $available[0]; - my $machine = $selected->{machine}; - - $slotLock = openSlotLock($machine, $selected->{free}); - flock($slotLock, LOCK_EX | LOCK_NB) or die; - utime undef, undef, $slotLock; - - close MAINLOCK; + # Prioritise the available machines as follows: + # - First by load divided by speed factor, rounded to the nearest + # integer. This causes fast machines to be preferred over slow + # machines with similar loads. + # - Then by speed factor. + # - Finally by load. + sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); } + @available = sort + { lf($a) <=> lf($b) + || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor} + || $a->{load} <=> $b->{load} + } @available; - # Connect to the selected machine. - @sshOpts = ("-i", $machine->{sshKeys}, "-x"); - $hostName = $machine->{hostName}; - last if openSSHConnection $hostName; + # Select the best available machine and lock a free slot. + my $selected = $available[0]; + my $machine = $selected->{machine}; + + $slotLock = openSlotLock($machine, $selected->{free}); + flock($slotLock, LOCK_EX | LOCK_NB) or die; + utime undef, undef, $slotLock; + + close MAINLOCK; + + + # Connect to the selected machine. + @sshOpts = ("-i", $machine->{sshKeys}, "-x"); + $hostName = $machine->{hostName}; + last REQ if openSSHConnection $hostName; - warn "unable to open SSH connection to $hostName, trying other available machines...\n"; - $machine->{enabled} = 0; + warn "unable to open SSH connection to $hostName, trying other available machines...\n"; + $machine->{enabled} = 0; + } } # Tell Nix we've accepted the build. -sendReply "accept"; -my $x = ; -chomp $x; - -if ($x ne "okay") { - exit 0; -} - - print STDERR "building `$drvPath' on `$hostName'\n"; +sendReply "accept"; +my @inputs = split /\s/, readline(STDIN); +my @outputs = split /\s/, readline(STDIN); -my @inputs = split /\s/, do { local $/; local @ARGV = "inputs"; <> }; -my @outputs = split /\s/, do { local $/; local @ARGV = "outputs"; <> }; my $maybeSign = ""; $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec"; @@ -238,7 +237,7 @@ if (system("ssh $hostName @sshOpts -tt 'nix-store -r $drvPath $buildFlags > /dev exit $res; } -print "build of `$drvPath' on `$hostName' succeeded\n"; +#print "build of `$drvPath' on `$hostName' succeeded\n"; # Copy the output from the build machine. @@ -246,7 +245,8 @@ foreach my $output (@outputs) { my $maybeSignRemote = ""; $maybeSignRemote = "--sign" if $UID != 0; - system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output' | @bindir@/nix-store --import > /dev/null") == 0 + system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" . + "| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0 or die "cannot copy $output from $hostName: $?"; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5617eccf1..f2781776c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -64,6 +64,7 @@ static const uid_t rootUserId = 0; /* Forward definition. */ class Worker; +class HookInstance; /* A pointer to a goal. */ @@ -215,6 +216,8 @@ public: LocalStore & store; + boost::shared_ptr hook; + Worker(LocalStore & store); ~Worker(); @@ -615,6 +618,94 @@ void deletePathWrapped(const Path & path) ////////////////////////////////////////////////////////////////////// +struct HookInstance +{ + /* Pipes for talking to the build hook. */ + Pipe toHook; + + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; + + /* The process ID of the hook. */ + Pid pid; + + HookInstance(); + + ~HookInstance(); +}; + + +HookInstance::HookInstance() +{ + debug("starting build hook"); + + Path buildHook = absPath(getEnv("NIX_BUILD_HOOK")); + + /* Create a pipe to get the output of the child. */ + fromHook.create(); + + /* Create the communication pipes. */ + toHook.create(); + + /* Fork the hook. */ + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + commonChildInit(fromHook); + + if (chdir("/") == -1) throw SysError("changing into `/"); + + /* Dup the communication pipes. */ + toHook.writeSide.close(); + if (dup2(toHook.readSide, STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); + + execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), + (format("%1%") % maxSilentTime).str().c_str(), NULL); + + throw SysError(format("executing `%1%'") % buildHook); + + } catch (std::exception & e) { + std::cerr << format("build hook error: %1%") % e.what() << std::endl; + } + quickExit(1); + } + + /* parent */ + pid.setSeparatePG(true); + pid.setKillSignal(SIGTERM); + fromHook.writeSide.close(); + toHook.readSide.close(); +} + + +HookInstance::~HookInstance() +{ + try { + /* Cleanly shut down the hook by closing its stdin if it's not + already building. Otherwise pid's destructor will kill + it. */ + if (pid != -1 && toHook.writeSide != -1) { + toHook.writeSide.close(); + pid.wait(true); + } + } catch (...) { + ignoreException(); + } +} + + +////////////////////////////////////////////////////////////////////// + + +typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; + class DerivationGoal : public Goal { private: @@ -651,12 +742,9 @@ private: /* Pipe for the builder's standard output/error. */ Pipe logPipe; - /* Whether we're building using a build hook. */ - bool usingBuildHook; - - /* Pipes for talking to the build hook (if any). */ - Pipe toHook; - + /* The build hook. */ + boost::shared_ptr hook; + /* Whether we're currently doing a chroot build. */ bool useChroot; @@ -694,12 +782,8 @@ private: void buildDone(); /* Is the build hook willing to perform the build? */ - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; HookReply tryBuildHook(); - /* Synchronously wait for a build hook to finish. */ - void terminateBuildHook(bool kill = false); - /* Start building a derivation. */ void startBuilder(); @@ -711,10 +795,6 @@ private: /* Open a log file and a pipe to it. */ Path openLogFile(); - /* Common initialisation to be performed in child processes (i.e., - both in builders and in build hooks). */ - void initChild(); - /* Delete the temporary directory, if we have one. */ void deleteTmpDir(bool force); @@ -742,6 +822,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker) trace("created"); } + DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a @@ -754,6 +835,7 @@ DerivationGoal::~DerivationGoal() } } + void DerivationGoal::killChild() { if (pid != -1) { @@ -778,6 +860,8 @@ void DerivationGoal::killChild() assert(pid == -1); } + + hook.reset(); } @@ -1048,7 +1132,6 @@ void DerivationGoal::tryToBuild() /* Is the build hook willing to accept this job? */ if (!preferLocalBuild) { - usingBuildHook = true; switch (tryBuildHook()) { case rpAccept: /* Yes, it has started doing so. Wait until we get @@ -1056,7 +1139,8 @@ void DerivationGoal::tryToBuild() state = &DerivationGoal::buildDone; return; case rpPostpone: - /* Not now; wait until at least one child finishes. */ + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -1066,8 +1150,6 @@ void DerivationGoal::tryToBuild() } } - usingBuildHook = false; - /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1108,16 +1190,23 @@ void DerivationGoal::buildDone() to have terminated. In fact, the builder could also have simply have closed its end of the pipe --- just don't do that :-) */ - /* !!! this could block! security problem! solution: kill the - child */ - pid_t savedPid = pid; - int status = pid.wait(true); + int status; + pid_t savedPid; + if (hook) { + savedPid = hook->pid; + status = hook->pid.wait(true); + } else { + /* !!! this could block! security problem! solution: kill the + child */ + savedPid = pid; + status = pid.wait(true); + } debug(format("builder process for `%1%' finished") % drvPath); /* So the child is gone now. */ worker.childTerminated(savedPid); - + /* Close the read side of the logger pipe. */ logPipe.readSide.close(); @@ -1192,11 +1281,11 @@ void DerivationGoal::buildDone() /* When using a build hook, the hook will return a remote build failure using exit code 100. Anything else is a hook problem. */ - bool hookError = usingBuildHook && + bool hookError = hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100); if (printBuildTrace) { - if (usingBuildHook && hookError) + if (hook && hookError) printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % status % e.msg()); else @@ -1231,162 +1320,74 @@ void DerivationGoal::buildDone() } -DerivationGoal::HookReply DerivationGoal::tryBuildHook() +HookReply DerivationGoal::tryBuildHook() { - if (!useBuildHook) return rpDecline; - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (buildHook == "") return rpDecline; - buildHook = absPath(buildHook); + if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; - /* Create a directory where we will store files used for - communication between us and the build hook. */ - tmpDir = createTempDir(); - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); + if (!worker.hook) + worker.hook = boost::shared_ptr(new HookInstance); - /* Create the communication pipes. */ - toHook.create(); - - /* Fork the hook. */ - pid = fork(); - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - initChild(); - - string s; - foreach (DerivationOutputs::const_iterator, i, drv.outputs) - s += i->second.path + " "; - if (setenv("NIX_HELD_LOCKS", s.c_str(), 1)) - throw SysError("setting an environment variable"); - - execl(buildHook.c_str(), buildHook.c_str(), - (worker.getNrLocalBuilds() < maxBuildJobs ? (string) "1" : "0").c_str(), - thisSystem.c_str(), - drv.platform.c_str(), - drvPath.c_str(), - (format("%1%") % maxSilentTime).str().c_str(), - NULL); - - throw SysError(format("executing `%1%'") % buildHook); - - } catch (std::exception & e) { - std::cerr << format("build hook error: %1%") % e.what() << std::endl; - } - quickExit(1); - } - - /* parent */ - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton >(logPipe.readSide), false, false); - - toHook.readSide.close(); + writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3%") % + (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") % drv.platform % drvPath).str()); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ string reply; - try { - while (true) { - string s = readLine(logPipe.readSide); - if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - handleChildOutput(logPipe.readSide, s + "\n"); + while (true) { + string s = readLine(worker.hook->fromHook.readSide); + if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; } - } catch (Error & e) { - terminateBuildHook(true); - throw; + handleChildOutput(worker.hook->fromHook.readSide, s + "\n"); } debug(format("hook reply is `%1%'") % reply); - if (reply == "decline" || reply == "postpone") { - /* Clean up the child. !!! hacky / should verify */ - terminateBuildHook(); + if (reply == "decline" || reply == "postpone") return reply == "decline" ? rpDecline : rpPostpone; - } + else if (reply != "accept") + throw Error(format("bad hook reply `%1%'") % reply); - else if (reply == "accept") { + printMsg(lvlTalkative, format("using hook to build path(s) %1%") + % showPaths(outputPaths(drv.outputs))); - printMsg(lvlInfo, format("using hook to build path(s) %1%") - % showPaths(outputPaths(drv.outputs))); + hook = worker.hook; + worker.hook.reset(); - /* Write the information that the hook needs to perform the - build, i.e., the set of input paths, the set of output - paths, and the references (pointer graph) in the input - paths. */ + /* Tell the hook all the inputs that have to be copied to the + remote system. This unfortunately has to contain the entire + derivation closure to ensure that the validity invariant holds + on the remote system. (I.e., it's unfortunate that we have to + list it since the remote system *probably* already has it.) */ + PathSet allInputs; + allInputs.insert(inputPaths.begin(), inputPaths.end()); + computeFSClosure(drvPath, allInputs); - Path inputListFN = tmpDir + "/inputs"; - Path outputListFN = tmpDir + "/outputs"; - Path referencesFN = tmpDir + "/references"; - - /* The `inputs' file lists all inputs that have to be copied - to the remote system. This unfortunately has to contain - the entire derivation closure to ensure that the validity - invariant holds on the remote system. (I.e., it's - unfortunate that we have to list it since the remote system - *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - computeFSClosure(drvPath, allInputs); + string s; + foreach (PathSet::iterator, i, allInputs) s += *i + " "; + writeLine(hook->toHook.writeSide, s); - string s; - foreach (PathSet::iterator, i, allInputs) s += *i + "\n"; + /* Tell the hooks the outputs that have to be copied back from the + remote system. */ + s = ""; + foreach (DerivationOutputs::iterator, i, drv.outputs) + s += i->second.path + " "; + writeLine(hook->toHook.writeSide, s); + + hook->toHook.writeSide.close(); + + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + worker.childStarted(shared_from_this(), + hook->pid, singleton >(hook->fromHook.readSide), false, false); + + if (printBuildTrace) + printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") + % drvPath % drv.outputs["out"].path % drv.platform % logFile); - writeFile(inputListFN, s); - - /* The `outputs' file lists all outputs that have to be copied - from the remote system. */ - s = ""; - foreach (DerivationOutputs::iterator, i, drv.outputs) - s += i->second.path + "\n"; - writeFile(outputListFN, s); - - /* The `references' file has exactly the format accepted by - `nix-store --register-validity'. */ - writeFile(referencesFN, - makeValidityRegistration(allInputs, true, false)); - - /* Tell the hook to proceed. */ - writeLine(toHook.writeSide, "okay"); - toHook.writeSide.close(); - - if (printBuildTrace) - printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") - % drvPath % drv.outputs["out"].path % drv.platform % logFile); - - return rpAccept; - } - - else throw Error(format("bad hook reply `%1%'") % reply); -} - - -void DerivationGoal::terminateBuildHook(bool kill) -{ - debug("terminating build hook"); - pid_t savedPid = pid; - if (kill) - pid.kill(); - else - pid.wait(true); - /* `false' means don't wake up waiting goals, since we want to - keep this build slot ourselves. */ - worker.childTerminated(savedPid, false); - toHook.writeSide.close(); - fdLogFile.close(); - logPipe.readSide.close(); - deleteTmpDir(true); /* get rid of the hook's temporary directory */ + return rpAccept; } @@ -1667,9 +1668,12 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("executing builder `%1%'") % drv.builder); - /* Create the log file and pipe. */ + /* Create the log file. */ Path logFile = openLogFile(); + /* Create a pipe to get the output of the child. */ + logPipe.create(); + /* Fork a child to build the package. Note that while we currently use forks to run and wait for the children, it shouldn't be hard to use threads for this on systems where @@ -1710,18 +1714,23 @@ void DerivationGoal::startBuilder() throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); } - /* Do the chroot(). initChild() will do a chdir() to - the temporary build directory to make sure the - current directory is in the chroot. (Actually the - order doesn't matter, since due to the bind mount - tmpDir and tmpRootDit/tmpDir are the same - directories.) */ + /* Do the chroot(). Below we do a chdir() to the + temporary build directory to make sure the current + directory is in the chroot. (Actually the order + doesn't matter, since due to the bind mount tmpDir + and tmpRootDit/tmpDir are the same directories.) */ if (chroot(chrootRootDir.c_str()) == -1) throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir); } #endif - initChild(); + commonChildInit(logPipe); + + if (chdir(tmpDir.c_str()) == -1) + throw SysError(format("changing into `%1%'") % tmpDir); + + /* Close all other file descriptors. */ + closeMostFDs(set()); #ifdef CAN_DO_LINUX32_BUILDS if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") { @@ -1742,10 +1751,10 @@ void DerivationGoal::startBuilder() /* If we are running in `build-users' mode, then switch to the user we allocated above. Make sure that we drop - all root privileges. Note that initChild() above has - closed all file descriptors except std*, so that's - safe. Also note that setuid() when run as root sets - the real, effective and saved UIDs. */ + all root privileges. Note that above we have closed + all file descriptors except std*, so that's safe. Also + note that setuid() when run as root sets the real, + effective and saved UIDs. */ if (buildUser.enabled()) { printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser()); @@ -1838,7 +1847,7 @@ void DerivationGoal::computeClosure() /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have to do anything here. */ - if (usingBuildHook) { + if (hook) { bool allValid = true; foreach (DerivationOutputs::iterator, i, drv.outputs) if (!worker.store.isValidPath(i->second.path)) allValid = false; @@ -1966,32 +1975,10 @@ Path DerivationGoal::openLogFile() if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); - /* Create a pipe to get the output of the child. */ - logPipe.create(); - return logFileName; } -void DerivationGoal::initChild() -{ - commonChildInit(logPipe); - - if (chdir(tmpDir.c_str()) == -1) - throw SysError(format("changing into `%1%'") % tmpDir); - - /* When running a hook, dup the communication pipes. */ - if (usingBuildHook) { - toHook.writeSide.close(); - if (dup2(toHook.readSide, STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - } - - /* Close all other file descriptors. */ - closeMostFDs(set()); -} - - void DerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { @@ -2011,19 +1998,16 @@ void DerivationGoal::deleteTmpDir(bool force) void DerivationGoal::handleChildOutput(int fd, const string & data) { - if (fd == logPipe.readSide) { - if (verbosity >= buildVerbosity) - writeToStderr((unsigned char *) data.c_str(), data.size()); + if (verbosity >= buildVerbosity) + writeToStderr((unsigned char *) data.c_str(), data.size()); + if (fdLogFile != -1) writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size()); - } - - else abort(); } void DerivationGoal::handleEOF(int fd) { - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/tests/build-hook.hook.sh b/tests/build-hook.hook.sh index 83fa3bf78..ff709985d 100755 --- a/tests/build-hook.hook.sh +++ b/tests/build-hook.hook.sh @@ -2,20 +2,22 @@ #set -x -drv=$4 +while read x y drv rest; do -echo "HOOK for $drv" >&2 + echo "HOOK for $drv" >&2 -outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv` + outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv` -echo "output path is $outPath" >&2 + echo "output path is $outPath" >&2 -if `echo $outPath | grep -q input-1`; then - echo "# accept" >&2 - read x - echo "got $x" - mkdir $outPath - echo "BAR" > $outPath/foo -else - echo "# decline" >&2 -fi + if `echo $outPath | grep -q input-1`; then + echo "# accept" >&2 + read inputs + read outputs + mkdir $outPath + echo "BAR" > $outPath/foo + else + echo "# decline" >&2 + fi + +done \ No newline at end of file From e41ecbf7303a181fd37026edb72f2f23dde7e73b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Aug 2010 11:09:04 +0000 Subject: [PATCH 079/128] --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 0b7f89448..81c008a84 100644 --- a/configure.ac +++ b/configure.ac @@ -240,7 +240,7 @@ AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], [prefix of SQLite]), sqlite=$withval, sqlite=) AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") -SQLITE_VERSION=3.7.0.1 +SQLITE_VERSION=3.7.2 AC_SUBST(SQLITE_VERSION) if test -z "$sqlite"; then sqlite_lib='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)/libsqlite3.la' From 766f7084188d8c0dd0595bbbfc50fbc4f9d67a50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Aug 2010 13:18:13 +0000 Subject: [PATCH 080/128] * Experimental feature: allow a derivation to tell the build hook that it requires a certain feature on the build machine, e.g. requiredSystemFeatures = [ "kvm" ]; We need this in Hydra to make sure that builds that require KVM support are forwarded to machines that have KVM support. Probably this should also be enforced for local builds. --- scripts/build-remote.pl.in | 27 +++++++++++++++++---------- src/libstore/build.cc | 12 ++++++++++-- src/libutil/util.cc | 11 +++++++++++ src/libutil/util.hh | 5 +++++ 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index a8f73f2e5..0c8081a0b 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -36,6 +36,8 @@ sub sendReply { print STDERR "# $reply\n"; } +sub all { $_ || return 0 for @_; 1 } + # Initialisation. my $loadIncreased = 0; @@ -64,13 +66,14 @@ if (defined $conf && -e $conf) { chomp; s/\#.*$//g; next if /^\s*$/; - /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)(\s+([0-9\.]+))?\s*$/ or die; + my @tokens = split /\s/, $_; push @machines, - { hostName => $1 - , systemTypes => [split(/,/, $2)] - , sshKeys => $3 - , maxJobs => $4 - , speedFactor => 1.0 * ($6 || 1) + { hostName => $tokens[0] + , systemTypes => [ split(/,/, $tokens[1]) ] + , sshKeys => $tokens[2] + , maxJobs => int($tokens[3]) + , speedFactor => 1.0 * (defined $tokens[4] ? int($tokens[4]) : 1) + , features => [ split(/,/, $tokens[5] || "") ] , enabled => 1 }; } @@ -85,7 +88,8 @@ my ($drvPath, $hostName, $slotLock); REQ: while (1) { $_ = || exit 0; my ($amWilling, $neededSystem); - ($amWilling, $neededSystem, $drvPath) = split; + ($amWilling, $neededSystem, $drvPath, $requiredFeatures) = split; + my @requiredFeatures = split /,/, $requiredFeatures; my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem); @@ -103,12 +107,15 @@ REQ: while (1) { while (1) { # Find all machine that can execute this build, i.e., that - # support builds for the given platform and are not at their - # job limit. + # support builds for the given platform and features, and are + # not at their job limit. my $rightType = 0; my @available = (); LOOP: foreach my $cur (@machines) { - if ($cur->{enabled} && grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) { + if ($cur->{enabled} + && (grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) + && all(map { my $f = $_; 0 != grep { $f eq $_ } @{$cur->{features}} } @requiredFeatures)) + { $rightType = 1; # We have a machine of the right type. Determine the load on diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f2781776c..f9c9a0a1e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1327,8 +1327,16 @@ HookReply DerivationGoal::tryBuildHook() if (!worker.hook) worker.hook = boost::shared_ptr(new HookInstance); - writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3%") % - (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") % drv.platform % drvPath).str()); + /* Tell the hook about system features (beyond the system type) + required from the build machine. (The hook could parse the + drv file itself, but this is easier.) */ + Strings features = tokenizeString(drv.env["requiredSystemFeatures"]); + foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ + + /* Send the request to the hook. */ + writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") + % (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") + % drv.platform % drvPath % concatStringsSep(",", features)).str()); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9540720fe..2d26fc66d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -966,6 +966,17 @@ Strings tokenizeString(const string & s, const string & separators) } +string concatStringsSep(const string & sep, const Strings & ss) +{ + string s; + foreach (Strings::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c45ebc062..27ad46904 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -280,6 +280,11 @@ MakeError(Interrupted, BaseError) Strings tokenizeString(const string & s, const string & separators = " \t\n\r"); +/* Concatenate the given strings with a separator between the + elements. */ +string concatStringsSep(const string & sep, const Strings & ss); + + /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); From 20acd43c25a388f5c31c2ee601f1cac88cf12f7b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Aug 2010 14:11:57 +0000 Subject: [PATCH 081/128] * Disable the GC reachability check for now (when creating new roots), as it's hopelessly inefficient. --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 87e0a05bc..b94442923 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -127,7 +127,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot, Instead of reading all the roots, it would be more efficient to check if the root is in a directory in or linked from the gcroots directory. */ - if (queryBoolSetting("gc-check-reachability", true)) { + if (queryBoolSetting("gc-check-reachability", false)) { Roots roots = store->findRoots(); if (roots.find(gcRoot) == roots.end()) printMsg(lvlError, From 80e722278ca03bf303961e2f27487dc98d042803 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Aug 2010 14:53:03 +0000 Subject: [PATCH 082/128] * When using the build hook, distinguish between the stderr of the hook script proper, and the stdout/stderr of the builder. Only the latter should be saved in /nix/var/log/nix/drvs. * Allow the verbosity to be set through an option. * Added a flag --quiet to lower the verbosity level. --- scripts/build-remote.pl.in | 4 ++-- scripts/nix-build.in | 7 ++++++- src/libmain/shared.cc | 10 +++++---- src/libstore/build.cc | 40 ++++++++++++++++++++++++++---------- src/libstore/gc.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/remote-store.cc | 9 +------- src/libutil/types.hh | 2 +- 8 files changed, 47 insertions(+), 29 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 0c8081a0b..8e3da2553 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -225,14 +225,14 @@ system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign # Perform the build. -my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsDir/\$PPID.out"; +my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsDir/\$PPID.out --option verbosity 0"; # `-tt' forces allocation of a pseudo-terminal. This is required to # make the remote nix-store process receive a signal when the # connection dies. Without it, the remote process might continue to # run indefinitely (that is, until it next tries to write to # stdout/stderr). -if (system("ssh $hostName @sshOpts -tt 'nix-store -r $drvPath $buildFlags > /dev/null'") != 0) { +if (system("ssh $hostName @sshOpts -tt 'nix-store -r $drvPath $buildFlags > /dev/null' >&4") != 0) { # If we couldn't run ssh or there was an ssh problem (indicated by # exit code 255), then we return exit code 1; otherwise we assume # that the builder failed, which we indicate to Nix using exit diff --git a/scripts/nix-build.in b/scripts/nix-build.in index ed85d5712..f9d81b36c 100644 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -123,6 +123,11 @@ EOF $verbose = 1; } + elsif ($arg eq "--quiet") { + push @buildArgs, $arg; + push @instArgs, $arg; + } + elsif (substr($arg, 0, 1) eq "-") { push @buildArgs, $arg; } @@ -165,7 +170,7 @@ foreach my $expr (@exprs) { # Build. my @outPaths; - $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-rv", + $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-r", @buildArgs, @drvPaths; while () {chomp; push @outPaths, $_;} if (!close OUTPATHS) { diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index eddc4e64b..f58def403 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -196,17 +196,16 @@ static void initAndRun(int argc, char * * argv) remaining.clear(); /* Process default options. */ + int verbosityDelta = 0; for (Strings::iterator i = args.begin(); i != args.end(); ++i) { string arg = *i; - if (arg == "--verbose" || arg == "-v") - verbosity = (Verbosity) ((int) verbosity + 1); + if (arg == "--verbose" || arg == "-v") verbosityDelta++; + else if (arg == "--quiet") verbosityDelta--; else if (arg == "--log-type") { ++i; if (i == args.end()) throw UsageError("`--log-type' requires an argument"); setLogType(*i); } - else if (arg == "--build-output" || arg == "-B") - ; /* !!! obsolete - remove eventually */ else if (arg == "--no-build-output" || arg == "-Q") buildVerbosity = lvlVomit; else if (arg == "--print-build-trace") @@ -247,6 +246,9 @@ static void initAndRun(int argc, char * * argv) else remaining.push_back(arg); } + verbosityDelta += queryIntSetting("verbosity", lvlInfo); + verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta); + /* Automatically clean up the temporary roots file when we exit. */ RemoveTempRoots removeTempRoots __attribute__((unused)); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f9c9a0a1e..30e37e4e9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -626,6 +626,9 @@ struct HookInstance /* Pipe for the hook's standard output/error. */ Pipe fromHook; + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + /* The process ID of the hook. */ Pid pid; @@ -647,6 +650,9 @@ HookInstance::HookInstance() /* Create the communication pipes. */ toHook.create(); + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + /* Fork the hook. */ pid = fork(); switch (pid) { @@ -666,6 +672,11 @@ HookInstance::HookInstance() if (dup2(toHook.readSide, STDIN_FILENO) == -1) throw SysError("dupping to-hook read side"); + /* Use fd 4 for the builder's stdout/stderr. */ + builderOut.readSide.close(); + if (dup2(builderOut.writeSide, 4) == -1) + throw SysError("dupping builder's stdout/stderr"); + execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), (format("%1%") % maxSilentTime).str().c_str(), NULL); @@ -740,7 +751,7 @@ private: AutoCloseFD fdLogFile; /* Pipe for the builder's standard output/error. */ - Pipe logPipe; + Pipe builderOut; /* The build hook. */ boost::shared_ptr hook; @@ -1208,7 +1219,11 @@ void DerivationGoal::buildDone() worker.childTerminated(savedPid); /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); + if (hook) { + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); + } + else builderOut.readSide.close(); /* Close the log file. */ fdLogFile.close(); @@ -1387,9 +1402,11 @@ HookReply DerivationGoal::tryBuildHook() /* Create the log file and pipe. */ Path logFile = openLogFile(); - - worker.childStarted(shared_from_this(), - hook->pid, singleton >(hook->fromHook.readSide), false, false); + + set fds; + fds.insert(hook->fromHook.readSide); + fds.insert(hook->builderOut.readSide); + worker.childStarted(shared_from_this(), hook->pid, fds, false, false); if (printBuildTrace) printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") @@ -1679,8 +1696,8 @@ void DerivationGoal::startBuilder() /* Create the log file. */ Path logFile = openLogFile(); - /* Create a pipe to get the output of the child. */ - logPipe.create(); + /* Create a pipe to get the output of the builder. */ + builderOut.create(); /* Fork a child to build the package. Note that while we currently use forks to run and wait for the children, it @@ -1732,7 +1749,7 @@ void DerivationGoal::startBuilder() } #endif - commonChildInit(logPipe); + commonChildInit(builderOut); if (chdir(tmpDir.c_str()) == -1) throw SysError(format("changing into `%1%'") % tmpDir); @@ -1816,9 +1833,9 @@ void DerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); - logPipe.writeSide.close(); + builderOut.writeSide.close(); worker.childStarted(shared_from_this(), pid, - singleton >(logPipe.readSide), true, true); + singleton >(builderOut.readSide), true, true); if (printBuildTrace) { printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") @@ -2008,7 +2025,8 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { if (verbosity >= buildVerbosity) writeToStderr((unsigned char *) data.c_str(), data.size()); - if (fdLogFile != -1) + if ((hook && fd == hook->builderOut.readSide) || + (!hook && fd == builderOut.readSide)) writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size()); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b94442923..ea784bcd9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -136,7 +136,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot, "therefore, `%2%' might be removed by the garbage collector") % gcRoot % storePath); } - + /* Grab the global GC root, causing us to block while a GC is in progress. This prevents the set of permanent roots from increasing while a GC is in progress. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 75d2f69c2..7069d104a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -20,7 +20,7 @@ string nixBinDir = "/UNINIT"; bool keepFailed = false; bool keepGoing = false; bool tryFallback = false; -Verbosity buildVerbosity = lvlInfo; +Verbosity buildVerbosity = lvlError; unsigned int maxBuildJobs = 1; unsigned int buildCores = 1; bool readOnlyMode = false; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8f162daee..92d517bbb 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -97,10 +97,6 @@ void RemoteStore::forkSlave() if (worker == "") worker = nixBinDir + "/nix-worker"; - string verbosityArg = "-"; - for (int i = 1; i < verbosity; ++i) - verbosityArg += "v"; - child = fork(); switch (child) { @@ -120,10 +116,7 @@ void RemoteStore::forkSlave() close(fdSocket); close(fdChild); - execlp(worker.c_str(), worker.c_str(), "--slave", - /* hacky - must be at the end */ - verbosityArg == "-" ? NULL : verbosityArg.c_str(), - NULL); + execlp(worker.c_str(), worker.c_str(), "--slave", NULL); throw SysError(format("executing `%1%'") % worker); diff --git a/src/libutil/types.hh b/src/libutil/types.hh index f110188da..86801342f 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -63,7 +63,7 @@ typedef set PathSet; typedef enum { - lvlError, + lvlError = 0, lvlInfo, lvlTalkative, lvlChatty, From e2e168f7c27f5239badf6e8705264bd907d6b82c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Aug 2010 11:47:31 +0000 Subject: [PATCH 083/128] `nix-store --verify' improvements: * If a path has disappeared, check its referrers first, and don't try to invalidate paths that have valid referrers. Otherwise we get a foreign key constraint violation. * Read the whole Nix store directory instead of statting each valid path, which is slower. * Acquire the global GC lock. --- src/libstore/gc.cc | 3 +- src/libstore/local-store.cc | 69 ++++++++++++++++++++++++++++--------- src/libstore/local-store.hh | 10 ++++-- 3 files changed, 62 insertions(+), 20 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ea784bcd9..7d58cd50a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,5 @@ #include "globals.hh" #include "misc.hh" -#include "pathlocks.hh" #include "local-store.hh" #include @@ -31,7 +30,7 @@ static const int defaultGcLevel = 1000; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -static int openGCLock(LockType lockType) +int LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % nixStateDir % gcLockName).str(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9254ceee4..fd6ad6464 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1226,27 +1226,25 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr void LocalStore::verifyStore(bool checkContents) { - /* Check whether all valid paths actually exist. */ - printMsg(lvlInfo, "checking path existence"); + printMsg(lvlError, format("reading the Nix store...")); - PathSet validPaths2 = queryValidPaths(), validPaths; + /* Acquire the global GC lock to prevent a garbage collection. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); - foreach (PathSet::iterator, i, validPaths2) { - checkInterrupt(); - /* !!! invalidatePath() will probably fail due to the foreign - key constraints on the Ref table. */ - if (!isStorePath(*i)) { - printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i); - invalidatePath(*i); - } else if (!pathExists(*i)) { - printMsg(lvlError, format("path `%1%' disappeared") % *i); - invalidatePath(*i); - } else validPaths.insert(*i); - } + Paths entries = readDirectory(nixStore); + PathSet store(entries.begin(), entries.end()); + + /* Check whether all valid paths actually exist. */ + printMsg(lvlInfo, "checking path existence..."); + + PathSet validPaths2 = queryValidPaths(), validPaths, done; + + foreach (PathSet::iterator, i, validPaths2) + verifyPath(*i, store, done, validPaths); /* Optionally, check the content hashes (slow). */ if (checkContents) { - printMsg(lvlInfo, "checking hashes"); + printMsg(lvlInfo, "checking hashes..."); foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i); @@ -1264,6 +1262,45 @@ void LocalStore::verifyStore(bool checkContents) } +void LocalStore::verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths) +{ + checkInterrupt(); + + if (done.find(path) != done.end()) return; + done.insert(path); + + if (!isStorePath(path)) { + printMsg(lvlError, format("path `%1%' is not in the Nix store") % path); + invalidatePath(path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; queryReferrers(path, referrers); + foreach (PathSet::iterator, i, referrers) + if (*i != path) { + verifyPath(*i, store, done, validPaths); + if (validPaths.find(*i) != validPaths.end()) + canInvalidate = false; + } + + if (canInvalidate) { + printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path); + invalidatePath(path); + } else + printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path); + + return; + } + + validPaths.insert(path); +} + + /* Functions for upgrading from the pre-SQLite database. */ PathSet LocalStore::queryValidPathsOld() diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3ae964f05..0d7ec1f49 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,6 +5,7 @@ #include "store-api.hh" #include "util.hh" +#include "pathlocks.hh" class sqlite3; @@ -16,8 +17,8 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.14. Version 6 is - Nix 0.15. */ + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + Nix 1.0. */ const int nixSchemaVersion = 6; @@ -233,6 +234,9 @@ private: void invalidatePath(const Path & path); + void verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths); + void upgradeStore6(); PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); @@ -244,6 +248,8 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); + int openGCLock(LockType lockType); + void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); From bf0dde959771661c6893001a7e0779b65d7be490 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Aug 2010 12:36:24 +0000 Subject: [PATCH 084/128] * Always print hook output on stderr, even if --no-build-output is set. * In the build hook, print a trace message to allow Hydra to pick up the name of the remote machine used for the build. --- scripts/build-remote.pl.in | 7 +++++-- src/libstore/build.cc | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 8e3da2553..65c9009b3 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -42,7 +42,7 @@ sub all { $_ || return 0 for @_; 1 } # Initialisation. my $loadIncreased = 0; -my ($localSystem, $maxSilentTime) = @ARGV; +my ($localSystem, $maxSilentTime, $printBuildTrace) = @ARGV; $maxSilentTime = 0 unless defined $maxSilentTime; my $currentLoad = $ENV{"NIX_CURRENT_LOAD"}; @@ -197,12 +197,15 @@ REQ: while (1) { # Tell Nix we've accepted the build. -print STDERR "building `$drvPath' on `$hostName'\n"; sendReply "accept"; my @inputs = split /\s/, readline(STDIN); my @outputs = split /\s/, readline(STDIN); +print STDERR "building `$drvPath' on `$hostName'\n"; +print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace; + + my $maybeSign = ""; $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec"; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 30e37e4e9..210486fbc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -678,7 +678,9 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), - (format("%1%") % maxSilentTime).str().c_str(), NULL); + (format("%1%") % maxSilentTime).str().c_str(), + (format("%1%") % printBuildTrace).str().c_str(), + NULL); throw SysError(format("executing `%1%'") % buildHook); @@ -1362,7 +1364,8 @@ HookReply DerivationGoal::tryBuildHook() reply = string(s, 2); break; } - handleChildOutput(worker.hook->fromHook.readSide, s + "\n"); + s += "\n"; + writeToStderr((unsigned char *) s.c_str(), s.size()); } debug(format("hook reply is `%1%'") % reply); @@ -2023,11 +2026,16 @@ void DerivationGoal::deleteTmpDir(bool force) void DerivationGoal::handleChildOutput(int fd, const string & data) { - if (verbosity >= buildVerbosity) - writeToStderr((unsigned char *) data.c_str(), data.size()); if ((hook && fd == hook->builderOut.readSide) || (!hook && fd == builderOut.readSide)) + { + if (verbosity >= buildVerbosity) + writeToStderr((unsigned char *) data.c_str(), data.size()); writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size()); + } + + if (hook && fd == hook->fromHook.readSide) + writeToStderr((unsigned char *) data.c_str(), data.size()); } From e4907411c2d902215d1a18456ce8b0c653650461 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Sep 2010 11:36:22 +0000 Subject: [PATCH 085/128] * Only do "pragma journal_mode = ..." if the current journal mode differs from the desired mode. There is an open SQLite ticket `Executing "PRAGMA journal_mode" may delete journal file while it is in use.' --- src/libstore/local-store.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fd6ad6464..c0c1461b4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -303,7 +303,16 @@ void LocalStore::openDB(bool create) The downside is that it doesn't work over NFS, so allow truncate mode alternatively. */ string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; - if (sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) + throw SQLiteError(db, "querying journal mode"); + prevMode = string((const char *) sqlite3_column_text(stmt, 0)); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting journal mode"); /* Increase the auto-checkpoint interval to 8192 pages. This From bfa6ee7d919b84a105f6376116e82240e44b990d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Oct 2010 12:30:46 +0000 Subject: [PATCH 086/128] * Don't use SSH's `-tt' flag because it doesn't seem to work on OpenSolaris when using connection sharing. Instead have the remote side check for disconnection and kill the process group when that happens. --- scripts/build-remote.pl.in | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 65c9009b3..f9bff9c41 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -4,6 +4,7 @@ use Fcntl ':flock'; use English '-no_match_vars'; use IO::Handle; use ssh qw/sshOpts openSSHConnection/; +no warnings('once'); # General operation: @@ -230,12 +231,15 @@ system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign # Perform the build. my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsDir/\$PPID.out --option verbosity 0"; -# `-tt' forces allocation of a pseudo-terminal. This is required to -# make the remote nix-store process receive a signal when the -# connection dies. Without it, the remote process might continue to -# run indefinitely (that is, until it next tries to write to -# stdout/stderr). -if (system("ssh $hostName @sshOpts -tt 'nix-store -r $drvPath $buildFlags > /dev/null' >&4") != 0) { +# We let the remote side kill its process group when the connection is +# closed unexpectedly. This is necessary to ensure that no processes +# are left running on the remote system if the local Nix process is +# killed. (SSH itself doesn't kill child processes if the connection +# is interrupted unless the `-tt' flag is used to force a pseudo-tty, +# in which case every child receives SIGHUP; however, `-tt' doesn't +# work on some platforms when connection sharing is used.) +pipe STDIN, DUMMY; # make sure we have a readable STDIN +if (system("ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) { # If we couldn't run ssh or there was an ssh problem (indicated by # exit code 255), then we return exit code 1; otherwise we assume # that the builder failed, which we indicate to Nix using exit From 64fd29855a8ae49cacdaff424679821b4fd3bf57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Oct 2010 15:55:51 +0000 Subject: [PATCH 087/128] * Wrap deleteFromStore() in a transaction. Otherwise there might be a race with other processes that add new referrers to a path, resulting in the garbage collector crashing with "foreign key constraint failed". (Nix/4) * Make --gc --print-dead etc. interruptible. --- src/libstore/gc.cc | 8 +++++--- src/libstore/local-store.cc | 13 ++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 7d58cd50a..b8395bfc4 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -421,7 +421,7 @@ struct LocalStore::GCState }; -static bool doDelete(GCOptions::GCAction action) +static bool shouldDelete(GCOptions::GCAction action) { return action == GCOptions::gcDeleteDead || action == GCOptions::gcDeleteSpecific; @@ -438,6 +438,8 @@ bool LocalStore::isActiveTempFile(const GCState & state, bool LocalStore::tryToDelete(GCState & state, const Path & path) { + checkInterrupt(); + if (!pathExists(path)) return true; if (state.deleted.find(path) != state.deleted.end()) return true; if (state.live.find(path) != state.live.end()) return false; @@ -516,7 +518,7 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) } /* The path is garbage, so delete it. */ - if (doDelete(state.options.action)) { + if (shouldDelete(state.options.action)) { printMsg(lvlInfo, format("deleting `%1%'") % path); unsigned long long bytesFreed, blocksFreed; @@ -625,7 +627,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) vector entries_(entries.begin(), entries.end()); random_shuffle(entries_.begin(), entries_.end()); - if (doDelete(state.options.action)) + if (shouldDelete(state.options.action)) printMsg(lvlError, format("deleting garbage...")); else printMsg(lvlError, format("determining live/dead paths...")); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0c1461b4..c0c75e34d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -298,11 +298,10 @@ void LocalStore::openDB(bool create) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting synchronous mode"); - /* Set the SQLite journal mode. The default is write-ahead - logging since it's the fastest and supports more concurrency. - The downside is that it doesn't work over NFS, so allow - truncate mode alternatively. */ - string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; + /* Set the SQLite journal mode. WAL mode is fastest, but doesn't + seem entirely stable at the moment (Oct. 2010). Thus, use + truncate mode by default. */ + string mode = queryBoolSetting("use-sqlite-wal", false) ? "wal" : "truncate"; string prevMode; { SQLiteStmt stmt; @@ -1220,6 +1219,8 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr assertStorePath(path); + SQLiteTxn txn(db); + if (isValidPath(path)) { PathSet referrers; queryReferrers(path, referrers); referrers.erase(path); /* ignore self-references */ @@ -1230,6 +1231,8 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr } deletePathWrapped(path, bytesFreed, blocksFreed); + + txn.commit(); } From a3883cbd28057a3dd2573f77dcda9a26faaac555 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Nov 2010 17:11:46 +0000 Subject: [PATCH 088/128] * Store the size of a store path in the database (to be precise, the size of the NAR serialisation of the path, i.e., `nix-store --dump PATH'). This is useful for Hydra. --- externals/Makefile.am | 2 +- src/libstore/build.cc | 41 +++++++++++-------- src/libstore/local-store.cc | 72 ++++++++++++++++++++++------------ src/libstore/local-store.hh | 9 ++--- src/libstore/optimise-store.cc | 2 +- src/libstore/references.cc | 2 +- src/libstore/references.hh | 2 +- src/libstore/remote-store.cc | 6 +++ src/libstore/remote-store.hh | 2 + src/libstore/schema.sql | 3 +- src/libstore/store-api.cc | 23 ++++++----- src/libstore/store-api.hh | 40 +++++++++++-------- src/libutil/hash.cc | 10 +++-- src/libutil/hash.hh | 7 ++-- src/nix-hash/nix-hash.cc | 2 +- src/nix-store/nix-store.cc | 12 +++--- 16 files changed, 144 insertions(+), 91 deletions(-) diff --git a/externals/Makefile.am b/externals/Makefile.am index 060ae2c52..bdc29fcb5 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -47,7 +47,7 @@ build-sqlite: else build-sqlite: $(SQLITE) (cd $(SQLITE) && \ - CC="$(CC)" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \ + CC="$(CC)" CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \ $(MAKE) ) touch build-sqlite diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 210486fbc..8b8be3e80 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1547,7 +1547,7 @@ void DerivationGoal::startBuilder() /* Write closure info to `fileName'. */ writeFile(tmpDir + "/" + fileName, - makeValidityRegistration(paths, false, false)); + worker.store.makeValidityRegistration(paths, false, false)); } @@ -1870,7 +1870,7 @@ PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) void DerivationGoal::computeClosure() { map allReferences; - map contentHashes; + map contentHashes; /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1927,7 +1927,7 @@ void DerivationGoal::computeClosure() if (ht == htUnknown) throw BuildError(format("unknown hash algorithm `%1%'") % algo); Hash h = parseHash(ht, i->second.hash); - Hash h2 = recursive ? hashPath(ht, path) : hashFile(ht, path); + Hash h2 = recursive ? hashPath(ht, path).first : hashFile(ht, path); if (h != h2) throw BuildError( format("output path `%1%' should have %2% hash `%3%', instead has `%4%'") @@ -1941,7 +1941,7 @@ void DerivationGoal::computeClosure() contained in it. Compute the SHA-256 NAR hash at the same time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ - Hash hash; + HashResult hash; PathSet references = scanForReferences(path, allPaths, hash); contentHashes[path] = hash; @@ -1970,14 +1970,18 @@ void DerivationGoal::computeClosure() } /* Register each output path as valid, and register the sets of - paths referenced by each of them. !!! this should be - atomic so that either all paths are registered as valid, or - none are. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.registerValidPath(i->second.path, - contentHashes[i->second.path], - allReferences[i->second.path], - drvPath); + paths referenced by each of them. */ + ValidPathInfos infos; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + ValidPathInfo info; + info.path = i->second.path; + info.hash = contentHashes[i->second.path].first; + info.narSize = contentHashes[i->second.path].second; + info.references = allReferences[i->second.path]; + info.deriver = drvPath; + infos.push_back(info); + } + worker.store.registerValidPaths(infos); /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not @@ -2385,10 +2389,15 @@ void SubstitutionGoal::finished() canonicalisePathMetaData(storePath); - Hash contentHash = hashPath(htSHA256, storePath); - - worker.store.registerValidPath(storePath, contentHash, - info.references, info.deriver); + HashResult hash = hashPath(htSHA256, storePath); + + ValidPathInfo info2; + info2.path = storePath; + info2.hash = hash.first; + info2.narSize = hash.second; + info2.references = info.references; + info2.deriver = info.deriver; + worker.store.registerValidPath(info2); outputLock->setDeletion(true); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0c75e34d..9595561bc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -327,9 +327,16 @@ void LocalStore::openDB(bool create) throw SQLiteError(db, "initialising database schema"); } + /* Backwards compatibility with old (pre-release) databases. Can + remove this eventually. */ + if (sqlite3_table_column_metadata(db, 0, "ValidPaths", "narSize", 0, 0, 0, 0, 0) != SQLITE_OK) { + if (sqlite3_exec(db, "alter table ValidPaths add column narSize integer" , 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "adding column narSize"); + } + /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -431,19 +438,6 @@ void canonicalisePathMetaData(const Path & path) } -void LocalStore::registerValidPath(const Path & path, - const Hash & hash, const PathSet & references, - const Path & deriver) -{ - ValidPathInfo info; - info.path = path; - info.hash = hash; - info.references = references; - info.deriver = deriver; - registerValidPath(info); -} - - unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) { SQLiteStmtUse use(stmtRegisterValidPath); @@ -454,6 +448,10 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) stmtRegisterValidPath.bind(info.deriver); else stmtRegisterValidPath.bind(); // null + if (info.narSize != 0) + stmtRegisterValidPath.bind(info.narSize); + else + stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); unsigned long long id = sqlite3_last_insert_rowid(db); @@ -920,10 +918,18 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, the path in the database. We may just have computed it above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ - registerValidPath(dstPath, - (recursive && hashAlgo == htSHA256) ? h : - (recursive ? hashString(htSHA256, dump) : hashPath(htSHA256, dstPath)), - PathSet(), ""); + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else + hash = hashPath(htSHA256, dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + registerValidPath(info); } outputLock.setDeletion(true); @@ -970,9 +976,15 @@ Path LocalStore::addTextToStore(const string & name, const string & s, writeFile(dstPath, s); canonicalisePathMetaData(dstPath); + + HashResult hash = hashPath(htSHA256, dstPath); - registerValidPath(dstPath, - hashPath(htSHA256, dstPath), references, ""); + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + registerValidPath(info); } outputLock.setDeletion(true); @@ -998,7 +1010,7 @@ struct HashAndWriteSink : Sink Hash currentHash() { HashSink hashSinkClone(hashSink); - return hashSinkClone.finish(); + return hashSinkClone.finish().first; } }; @@ -1136,7 +1148,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path deriver = readString(hashAndReadSource); if (deriver != "") assertStorePath(deriver); - Hash hash = hashAndReadSource.hashSink.finish(); + Hash hash = hashAndReadSource.hashSink.finish().first; hashAndReadSource.hashing = false; bool haveSignature = readInt(hashAndReadSource) == 1; @@ -1200,9 +1212,15 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ - if (deriver != "" && !isValidPath(deriver)) deriver = ""; - registerValidPath(dstPath, - hashPath(htSHA256, dstPath), references, deriver); + HashResult hash = hashPath(htSHA256, dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; + registerValidPath(info); } outputLock.setDeletion(true); @@ -1263,12 +1281,14 @@ void LocalStore::verifyStore(bool checkContents) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); - Hash current = hashPath(info.hash.type, *i); + Hash current = hashPath(info.hash.type, *i).first; if (current != info.hash) { printMsg(lvlError, format("path `%1%' was modified! " "expected hash `%2%', got `%3%'") % *i % printHash(info.hash) % printHash(current)); } + + /* !!! Check info.narSize */ } } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0d7ec1f49..2fd31c26d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -97,6 +97,8 @@ public: PathSet queryValidPaths(); + ValidPathInfo queryPathInfo(const Path & path); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); @@ -173,8 +175,7 @@ public: execution of the derivation (or something equivalent). Also register the hash of the file system contents of the path. The hash must be a SHA-256 hash. */ - void registerValidPath(const Path & path, - const Hash & hash, const PathSet & references, const Path & deriver); + void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); @@ -224,10 +225,6 @@ private: void addReference(unsigned long long referrer, unsigned long long reference); - void registerValidPath(const ValidPathInfo & info); - - ValidPathInfo queryPathInfo(const Path & path); - void appendReferrer(const Path & from, const Path & to, bool lock); void rewriteReferrers(const Path & path, bool purge, PathSet referrers); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 3ed54e24d..89be6ac65 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -68,7 +68,7 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath, the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path); + Hash hash = hashPath(htSHA256, path).first; stats.totalFiles++; printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a6f6e85fc..ade9c9aa2 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -81,7 +81,7 @@ void RefScanSink::operator () (const unsigned char * data, unsigned int len) PathSet scanForReferences(const string & path, - const PathSet & refs, Hash & hash) + const PathSet & refs, HashResult & hash) { RefScanSink sink; std::map backMap; diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 7d068eb51..158c08a77 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,7 +7,7 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs, - Hash & hash); + HashResult & hash); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 92d517bbb..517c886b4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -247,6 +247,12 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, } +ValidPathInfo RemoteStore::queryPathInfo(const Path & path) +{ + throw Error("not implemented"); +} + + Hash RemoteStore::queryPathHash(const Path & path) { openConnection(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 02a1c4752..519f46fd1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,6 +29,8 @@ public: PathSet queryValidPaths(); + ValidPathInfo queryPathInfo(const Path & path); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 7438632ed..c1b4a689a 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -3,7 +3,8 @@ create table if not exists ValidPaths ( path text unique not null, hash text not null, registrationTime integer not null, - deriver text + deriver text, + narSize integer ); create table if not exists Refs ( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 01dd51621..4b04f5751 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -190,7 +190,7 @@ std::pair computeStorePathForPath(const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) { HashType ht(hashAlgo); - Hash h = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath); + Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath); string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); return std::pair(dstPath, h); @@ -216,7 +216,7 @@ Path computeStorePathForText(const string & name, const string & s, /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string makeValidityRegistration(const PathSet & paths, +string StoreAPI::makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash) { string s = ""; @@ -224,18 +224,19 @@ string makeValidityRegistration(const PathSet & paths, foreach (PathSet::iterator, i, paths) { s += *i + "\n"; - if (showHash) - s += printHash(store->queryPathHash(*i)) + "\n"; + ValidPathInfo info = queryPathInfo(*i); - Path deriver = showDerivers ? store->queryDeriver(*i) : ""; + if (showHash) { + s += printHash(info.hash) + "\n"; + s += (format("%1%\n") % info.narSize).str(); + } + + Path deriver = showDerivers ? info.deriver : ""; s += deriver + "\n"; - PathSet references; - store->queryReferences(*i, references); + s += (format("%1%\n") % info.references.size()).str(); - s += (format("%1%\n") % references.size()).str(); - - foreach (PathSet::iterator, j, references) + foreach (PathSet::iterator, j, info.references) s += *j + "\n"; } @@ -252,6 +253,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string s; getline(str, s); info.hash = parseHash(htSHA256, s); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); } getline(str, info.deriver); string s; int n; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 082a9edc4..b0071da83 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -90,6 +90,21 @@ struct SubstitutablePathInfo }; +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; + time_t registrationTime; + unsigned long long narSize; // 0 = unknown + unsigned long long id; // internal use only + ValidPathInfo() : registrationTime(0), narSize(0) { } +}; + +typedef list ValidPathInfos; + + class StoreAPI { public: @@ -102,6 +117,9 @@ public: /* Query the set of valid paths. */ virtual PathSet queryValidPaths() = 0; + /* Query information about a valid path. */ + virtual ValidPathInfo queryPathInfo(const Path & path) = 0; + /* Queries the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; @@ -214,6 +232,12 @@ public: /* Clear the "failed" status of the given paths. The special value `*' causes all failed paths to be cleared. */ virtual void clearFailedPaths(const PathSet & paths) = 0; + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash); }; @@ -307,22 +331,6 @@ boost::shared_ptr openStore(); string showPaths(const PathSet & paths); -string makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash); - -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash hash; - PathSet references; - time_t registrationTime; - unsigned long long id; // internal use only - ValidPathInfo() : registrationTime(0) { } -}; - -typedef list ValidPathInfos; - ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index bd7e33a48..b9e784699 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -286,12 +286,14 @@ Hash hashFile(HashType ht, const Path & path) HashSink::HashSink(HashType ht) : ht(ht) { ctx = new Ctx; + bytes = 0; start(ht, *ctx); } HashSink::HashSink(const HashSink & h) { ht = h.ht; + bytes = h.bytes; ctx = new Ctx; *ctx = *h.ctx; } @@ -304,18 +306,20 @@ HashSink::~HashSink() void HashSink::operator () (const unsigned char * data, unsigned int len) { + bytes += len; update(ht, *ctx, data, len); } -Hash HashSink::finish() +HashResult HashSink::finish() { Hash hash(ht); nix::finish(ht, *ctx, hash.hash); - return hash; + return HashResult(hash, bytes); } -Hash hashPath(HashType ht, const Path & path, PathFilter & filter) +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) { HashSink sink(ht); dumpPath(path, sink, filter); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 81425b234..13740954d 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -40,7 +40,6 @@ struct Hash /* For sorting. */ bool operator < (const Hash & h) const; - }; @@ -72,7 +71,8 @@ Hash hashFile(HashType ht, const Path & path); (essentially) hashString(ht, dumpPath(path)). */ struct PathFilter; extern PathFilter defaultPathFilter; -Hash hashPath(HashType ht, const Path & path, +typedef std::pair HashResult; +HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); /* Compress a hash to the specified number of bytes by cyclically @@ -93,13 +93,14 @@ class HashSink : public Sink private: HashType ht; Ctx * ctx; + unsigned long long bytes; public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); virtual void operator () (const unsigned char * data, unsigned int len); - Hash finish(); + HashResult finish(); }; diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc index 8a6655403..4867234bf 100644 --- a/src/nix-hash/nix-hash.cc +++ b/src/nix-hash/nix-hash.cc @@ -44,7 +44,7 @@ void run(Strings args) if (op == opHash) { for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { - Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i); + Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i).first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % (base32 ? printHash32(h) : printHash(h)); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 695eb10dc..120f6ce72 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -393,9 +393,8 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); PathSet validPaths = store->queryValidPaths(); - foreach (PathSet::iterator, i, validPaths) { - cout << makeValidityRegistration(singleton(*i), true, true); - } + foreach (PathSet::iterator, i, validPaths) + cout << store->makeValidityRegistration(singleton(*i), true, true); } @@ -410,8 +409,11 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) /* !!! races */ if (canonicalise) canonicalisePathMetaData(info.path); - if (!hashGiven) - info.hash = hashPath(htSHA256, info.path); + if (!hashGiven) { + HashResult hash = hashPath(htSHA256, info.path); + info.hash = hash.first; + info.narSize = hash.second; + } infos.push_back(info); } } From 1db6259076b1b8f667451da8d2e44a55ece19056 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 12:08:01 +0000 Subject: [PATCH 089/128] * Implement RemoteStore::queryPathInfo(). --- src/libstore/remote-store.cc | 14 +++++++++++++- src/libstore/worker-protocol.hh | 1 + src/nix-worker/nix-worker.cc | 13 +++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 517c886b4..08969a623 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -249,7 +249,19 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, ValidPathInfo RemoteStore::queryPathInfo(const Path & path) { - throw Error("not implemented"); + openConnection(); + writeInt(wopQueryPathInfo, to); + writeString(path, to); + processStderr(); + ValidPathInfo info; + info.path = path; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.hash = parseHash(htSHA256, readString(from)); + info.references = readStorePaths(from); + info.registrationTime = readInt(from); + info.narSize = readLongLong(from); + return info; } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index a4dc690b7..2764f82c2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -38,6 +38,7 @@ typedef enum { wopQueryValidPaths = 23, wopQueryFailedPaths = 24, wopClearFailedPaths = 25, + wopQueryPathInfo = 26, } WorkerOp; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 9be733d8c..dbcd90be1 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -550,6 +550,19 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryPathInfo: { + Path path = readStorePath(from); + startWork(); + ValidPathInfo info = store->queryPathInfo(path); + stopWork(); + writeString(info.deriver, to); + writeString(printHash(info.hash), to); + writeStringSet(info.references, to); + writeInt(info.registrationTime, to); + writeLongLong(info.narSize, to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } From e60c962fb8dd3d8be37c1f4ae08d5247901fa129 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 12:40:52 +0000 Subject: [PATCH 090/128] * Add an operation `nix-store -q --size'. --- doc/manual/nix-store.xml | 19 ++++++++++++++++--- src/libstore/local-store.cc | 14 ++++++++++++-- src/libstore/local-store.hh | 1 + src/nix-store/help.txt | 3 ++- src/nix-store/nix-store.cc | 13 +++++++++---- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/doc/manual/nix-store.xml b/doc/manual/nix-store.xml index 10bb3eda5..a32559c03 100644 --- a/doc/manual/nix-store.xml +++ b/doc/manual/nix-store.xml @@ -404,6 +404,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' name + @@ -587,9 +588,21 @@ query is applied to the target of the symlink. Prints the SHA-256 hash of the contents of the - store paths paths. Since the hash is - stored in the Nix database, this is a fast - operation. + store paths paths (that is, the hash of + the output of nix-store --dump on the given + paths). Since the hash is stored in the Nix database, this is a + fast operation. + + + + + + Prints the size in bytes of the contents of the + store paths paths — to be precise, the + size of the output of nix-store --dump on the + given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9595561bc..201e4bae1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -88,6 +88,13 @@ void SQLiteStmt::bind(int value) } +void SQLiteStmt::bind64(long long value) +{ + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + void SQLiteStmt::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) @@ -340,7 +347,7 @@ void LocalStore::openDB(bool create) stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); + "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); stmtQueryReferrers.create(db, @@ -449,7 +456,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) else stmtRegisterValidPath.bind(); // null if (info.narSize != 0) - stmtRegisterValidPath.bind(info.narSize); + stmtRegisterValidPath.bind64(info.narSize); else stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) @@ -600,6 +607,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); if (s) info.deriver = s; + /* Note that narSize = NULL yields 0. */ + info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + /* Get the references. */ SQLiteStmtUse use2(stmtQueryReferences); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 2fd31c26d..4076e5957 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -70,6 +70,7 @@ struct SQLiteStmt operator sqlite3_stmt * () { return stmt; } void bind(const string & value); void bind(int value); + void bind64(long long value); void bind(); }; diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index 342281b33..478251851 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -16,7 +16,7 @@ Operations: --gc: run the garbage collector - --dump: dump a path as a Nix archive, forgetting dependencies + --dump: dump a path as a Nix archive (NAR), forgetting dependencies --restore: restore a path from a Nix archive, without registering validity @@ -44,6 +44,7 @@ Query flags: --graph: print a dot graph rooted at given path --xml: emit an XML representation of the graph rooted at the given path --hash: print the SHA-256 hash of the contents of the path + --size: print the size of the NAR dump of the path --roots: print the garbage collector roots that point to the path Query switches (not applicable to all queries): diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 120f6ce72..49a705585 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -226,7 +226,7 @@ static void printTree(const Path & path, static void opQuery(Strings opFlags, Strings opArgs) { enum { qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash + , qReferrersClosure, qDeriver, qBinding, qHash, qSize , qTree, qGraph, qXml, qResolve, qRoots } query = qOutputs; bool useOutput = false; bool includeOutputs = false; @@ -248,6 +248,7 @@ static void opQuery(Strings opFlags, Strings opArgs) query = qBinding; } else if (*i == "--hash") query = qHash; + else if (*i == "--size") query = qSize; else if (*i == "--tree") query = qTree; else if (*i == "--graph") query = qGraph; else if (*i == "--xml") query = qXml; @@ -310,11 +311,15 @@ static void opQuery(Strings opFlags, Strings opArgs) break; case qHash: + case qSize: foreach (Strings::iterator, i, opArgs) { Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise); - Hash hash = store->queryPathHash(path); - assert(hash.type == htSHA256); - cout << format("sha256:%1%\n") % printHash32(hash); + ValidPathInfo info = store->queryPathInfo(path); + if (query == qHash) { + assert(info.hash.type == htSHA256); + cout << format("sha256:%1%\n") % printHash32(info.hash); + } else if (query == qSize) + cout << format("%1%\n") % info.narSize; } break; From 5693b8a7e246af352c26c258e2af6402601839ed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 12:51:54 +0000 Subject: [PATCH 091/128] * nix-push: no need to compute the NAR hash, since the Nix database already has it (`nix-store -q --hash'). --- corepkgs/nar/nar.sh.in | 2 -- scripts/nix-push.in | 12 +++++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/corepkgs/nar/nar.sh.in b/corepkgs/nar/nar.sh.in index 67933ac67..1369d3a21 100644 --- a/corepkgs/nar/nar.sh.in +++ b/corepkgs/nar/nar.sh.in @@ -7,8 +7,6 @@ dst=$out/tmp.nar.bz2 @bzip2@ < tmp > $dst -@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash - @bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash @coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2 diff --git a/scripts/nix-push.in b/scripts/nix-push.in index 38097f740..c7da528b5 100644 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -172,12 +172,6 @@ for (my $n = 0; $n < scalar @storePaths; $n++) { $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash"; close HASH; - open HASH, "$narDir/nar-hash" or die "cannot open nar-hash"; - my $narHash = ; - chomp $narHash; - $narHash =~ /^[0-9a-z]+$/ or die "invalid hash"; - close HASH; - my $narName = "$narbz2Hash.nar.bz2"; my $narFile = "$narDir/$narName"; @@ -195,6 +189,10 @@ for (my $n = 0; $n < scalar @storePaths; $n++) { chomp $deriver; $deriver = "" if $deriver eq "unknown-deriver"; + my $narHash = `$binDir/nix-store --query --hash '$storePath'`; + die "cannot query hash for `$storePath'" if $? != 0; + chomp $narHash; + my $url; if ($localCopy) { $url = "$targetArchivesUrl/$narName"; @@ -205,7 +203,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) { { url => $url , hash => "$hashAlgo:$narbz2Hash" , size => $narbz2Size - , narHash => "$hashAlgo:$narHash" + , narHash => "$narHash" , references => $references , deriver => $deriver } From 06699d4219019182d9e45ebc613ae1a1df23f257 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 12:57:52 +0000 Subject: [PATCH 092/128] * Store the NAR size in the manifest. --- scripts/nix-push.in | 5 +++++ scripts/readmanifest.pm.in | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/nix-push.in b/scripts/nix-push.in index c7da528b5..6760baa7e 100644 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -193,6 +193,10 @@ for (my $n = 0; $n < scalar @storePaths; $n++) { die "cannot query hash for `$storePath'" if $? != 0; chomp $narHash; + my $narSize = `$binDir/nix-store --query --size '$storePath'`; + die "cannot query size for `$storePath'" if $? != 0; + chomp $narSize; + my $url; if ($localCopy) { $url = "$targetArchivesUrl/$narName"; @@ -204,6 +208,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) { , hash => "$hashAlgo:$narbz2Hash" , size => $narbz2Size , narHash => "$narHash" + , narSize => $narSize , references => $references , deriver => $deriver } diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index 7244984ea..2e6968c7d 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -165,8 +165,9 @@ sub writeManifest { print MANIFEST " StorePath: $storePath\n"; print MANIFEST " NarURL: $narFile->{url}\n"; print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; - print MANIFEST " NarHash: $narFile->{narHash}\n"; print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; + print MANIFEST " NarHash: $narFile->{narHash}\n"; + print MANIFEST " NarSize: $narFile->{narSize}\n" if $narFile->{narSize}; print MANIFEST " References: $narFile->{references}\n" if defined $narFile->{references} && $narFile->{references} ne ""; print MANIFEST " Deriver: $narFile->{deriver}\n" From bdf089f46362b8c9defefa0a14e3198582e12818 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 14:31:42 +0000 Subject: [PATCH 093/128] * Before a build, show the disk space that the downloaded store paths will approximately require. --- scripts/copy-from-other-stores.pl.in | 7 ++++++- scripts/download-using-manifests.pl.in | 2 ++ scripts/readmanifest.pm.in | 26 ++++++++++---------------- src/libmain/shared.cc | 13 +++++++------ src/libstore/local-store.cc | 1 + src/libstore/misc.cc | 5 +++-- src/libstore/misc.hh | 2 +- src/libstore/remote-store.cc | 4 ++-- src/libstore/store-api.hh | 1 + src/libstore/worker-protocol.hh | 2 +- src/nix-worker/nix-worker.cc | 2 ++ 11 files changed, 36 insertions(+), 29 deletions(-) diff --git a/scripts/copy-from-other-stores.pl.in b/scripts/copy-from-other-stores.pl.in index a6a14c3dc..10130c089 100644 --- a/scripts/copy-from-other-stores.pl.in +++ b/scripts/copy-from-other-stores.pl.in @@ -63,10 +63,15 @@ if ($ARGV[0] eq "--query") { `@bindir@/nix-store --query --references $storePath`; die "cannot query references of `$storePath'" if $? != 0; + my $narSize = `@bindir@/nix-store --query --size $storePath`; + die "cannot query size of `$storePath'" if $? != 0; + chomp $narSize; + print "$deriver\n"; print scalar @references, "\n"; print "$_\n" foreach @references; - print "0\n"; # !!! showing size not supported (yet) + print "$narSize\n"; + print "$narSize\n"; } else { die "unknown command `$cmd'"; } diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index d48c7dd4b..7b3f7ee66 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -64,6 +64,8 @@ if ($ARGV[0] eq "--query") { print "$_\n" foreach @references; my $size = $info->{size} || 0; print "$size\n"; + my $narSize = $info->{narSize} || 0; + print "$narSize\n"; } else { die "unknown command `$cmd'"; } diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index 2e6968c7d..c2c4be89b 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -33,18 +33,8 @@ sub readManifest { my $manifestVersion = 2; - my $storePath; - my $url; - my $hash; - my $size; - my $basePath; - my $baseHash; - my $patchType; - my $narHash; - my $references; - my $deriver; - my $hashAlgo; - my $copyFrom; + my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); + my ($narHash, $narSize, $references, $deriver, $hashAlgo, $copyFrom); while () { chomp; @@ -62,6 +52,7 @@ sub readManifest { undef $hash; undef $size; undef $narHash; + undef $narSize; undef $basePath; undef $baseHash; undef $patchType; @@ -89,7 +80,8 @@ sub readManifest { if (!$found) { push @{$narFileList}, { url => $url, hash => $hash, size => $size - , narHash => $narHash, references => $references + , narHash => $narHash, narSize => $narSize + , references => $references , deriver => $deriver, hashAlgo => $hashAlgo }; } @@ -100,8 +92,8 @@ sub readManifest { addPatch $patches, $storePath, { url => $url, hash => $hash, size => $size , basePath => $basePath, baseHash => $baseHash - , narHash => $narHash, patchType => $patchType - , hashAlgo => $hashAlgo + , narHash => $narHash, narSize => $narSize + , patchType => $patchType, hashAlgo => $hashAlgo }; } @@ -132,6 +124,7 @@ sub readManifest { elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } + elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; } elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } @@ -183,8 +176,9 @@ sub writeManifest { print MANIFEST " StorePath: $storePath\n"; print MANIFEST " NarURL: $patch->{url}\n"; print MANIFEST " Hash: $patch->{hash}\n"; - print MANIFEST " NarHash: $patch->{narHash}\n"; print MANIFEST " Size: $patch->{size}\n"; + print MANIFEST " NarHash: $patch->{narHash}\n"; + print MANIFEST " NarSize: $patch->{narSize}\n" if $patch->{narSize}; print MANIFEST " BasePath: $patch->{basePath}\n"; print MANIFEST " BaseHash: $patch->{baseHash}\n"; print MANIFEST " Type: $patch->{patchType}\n"; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d7879f035..7c2d92030 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -54,25 +54,26 @@ void printGCWarning() void printMissing(const PathSet & paths) { - unsigned long long downloadSize; + unsigned long long downloadSize, narSize; PathSet willBuild, willSubstitute, unknown; - queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize); + queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty()) { - printMsg(lvlInfo, format("the following derivations will be built:")); + printMsg(lvlInfo, format("these derivations will be built:")); foreach (PathSet::iterator, i, willBuild) printMsg(lvlInfo, format(" %1%") % *i); } if (!willSubstitute.empty()) { - printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") % - (downloadSize / (1024.0 * 1024.0))); + printMsg(lvlInfo, format("these paths will be downloaded/copied (%.2f MiB download, %.2f MiB unpacked):") + % (downloadSize / (1024.0 * 1024.0)) + % (narSize / (1024.0 * 1024.0))); foreach (PathSet::iterator, i, willSubstitute) printMsg(lvlInfo, format(" %1%") % *i); } if (!unknown.empty()) { - printMsg(lvlInfo, format("don't know how to build the following paths%1%:") + printMsg(lvlInfo, format("don't know how to build these paths%1%:") % (readOnlyMode ? " (may be caused by read-only store access)" : "")); foreach (PathSet::iterator, i, unknown) printMsg(lvlInfo, format(" %1%") % *i); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 201e4bae1..37bbbfdad 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -839,6 +839,7 @@ bool LocalStore::querySubstitutablePathInfo(const Path & substituter, info.references.insert(p); } info.downloadSize = getIntLine(run.from); + info.narSize = getIntLine(run.from); return true; } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index d52dd6346..01d6a97ae 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -48,9 +48,9 @@ Path findOutput(const Derivation & drv, string id) void queryMissing(const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize) + unsigned long long & downloadSize, unsigned long long & narSize) { - downloadSize = 0; + downloadSize = narSize = 0; PathSet todo(targets.begin(), targets.end()), done; @@ -88,6 +88,7 @@ void queryMissing(const PathSet & targets, if (store->querySubstitutablePathInfo(p, info)) { willSubstitute.insert(p); downloadSize += info.downloadSize; + narSize += info.narSize; todo.insert(info.references.begin(), info.references.end()); } else unknown.insert(p); diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh index 0bc9a2ee0..abef6237e 100644 --- a/src/libstore/misc.hh +++ b/src/libstore/misc.hh @@ -31,7 +31,7 @@ Path findOutput(const Derivation & drv, string id); will be substituted. */ void queryMissing(const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize); + unsigned long long & downloadSize, unsigned long long & narSize); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 08969a623..26093a5d3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -191,9 +191,8 @@ void RemoteStore::setOptions() writeInt(logType, to); writeInt(printBuildTrace, to); } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) { + if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) writeInt(buildCores, to); - } processStderr(); } @@ -243,6 +242,7 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, if (info.deriver != "") assertStorePath(info.deriver); info.references = readStorePaths(from); info.downloadSize = readLongLong(from); + info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; return true; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b0071da83..40ac88714 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -87,6 +87,7 @@ struct SubstitutablePathInfo Path deriver; PathSet references; unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ }; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 2764f82c2..d77049bc7 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x106 +#define PROTOCOL_VERSION 0x107 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index dbcd90be1..081b5b2fd 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -521,6 +521,8 @@ static void performOp(unsigned int clientVersion, writeString(info.deriver, to); writeStringSet(info.references, to); writeLongLong(info.downloadSize, to); + if (GET_PROTOCOL_MINOR(clientVersion) >= 7) + writeLongLong(info.narSize, to); } break; } From 1e24cbaba3a7b553569c72afddeb825001ae1dd7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 14:35:14 +0000 Subject: [PATCH 094/128] * Fix the test. --- tests/substituter.sh | 1 + tests/substituter2.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/substituter.sh b/tests/substituter.sh index 94ae26472..8985982c2 100755 --- a/tests/substituter.sh +++ b/tests/substituter.sh @@ -17,6 +17,7 @@ if test $1 = "--query"; then echo "" # deriver echo 0 # nr of refs echo 0 # download size + echo 0 # nar size else echo "bad command $cmd" exit 1 diff --git a/tests/substituter2.sh b/tests/substituter2.sh index 70f7d0943..c56a1bc47 100755 --- a/tests/substituter2.sh +++ b/tests/substituter2.sh @@ -16,6 +16,7 @@ if test $1 = "--query"; then echo "" # deriver echo 0 # nr of refs echo 0 # download size + echo 0 # nar size else echo "bad command $cmd" exit 1 From bf658f016ff16af79535358c8e609a933b86e6d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 14:39:02 +0000 Subject: [PATCH 095/128] * Test that download sizes are shown correctly. --- tests/substituter.sh | 4 ++-- tests/substitutes.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/substituter.sh b/tests/substituter.sh index 8985982c2..65c4fa185 100755 --- a/tests/substituter.sh +++ b/tests/substituter.sh @@ -16,8 +16,8 @@ if test $1 = "--query"; then echo 1 echo "" # deriver echo 0 # nr of refs - echo 0 # download size - echo 0 # nar size + echo $((1 * 1024 * 1024)) # download size + echo $((2 * 1024 * 1024)) # nar size else echo "bad command $cmd" exit 1 diff --git a/tests/substitutes.sh b/tests/substitutes.sh index 042827314..b48576c8c 100644 --- a/tests/substitutes.sh +++ b/tests/substitutes.sh @@ -14,6 +14,8 @@ echo $outPath > $TEST_ROOT/sub-paths export NIX_SUBSTITUTERS=$(pwd)/substituter.sh +$nixstore -r "$drvPath" --dry-run 2>&1 | grep -q "1.00 MiB.*2.00 MiB" + $nixstore -rvv "$drvPath" text=$(cat "$outPath"/hello) From a07c68f05edd754e389e0757ed2deefd70aad364 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 15:30:07 +0000 Subject: [PATCH 096/128] * Finally, a test for the binary patch functionality. --- tests/Makefile.am | 3 ++- tests/binary-patching.nix | 15 +++++++++++++++ tests/binary-patching.sh | 33 +++++++++++++++++++++++++++++++++ tests/init.sh | 6 +++++- tests/nix-push.sh | 2 +- 5 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 tests/binary-patching.nix create mode 100644 tests/binary-patching.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index beb7852db..88925fbd9 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -7,7 +7,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ - remote-store.sh export.sh export-graph.sh negative-caching.sh + remote-store.sh export.sh export-graph.sh negative-caching.sh \ + binary-patching.sh XFAIL_TESTS = diff --git a/tests/binary-patching.nix b/tests/binary-patching.nix new file mode 100644 index 000000000..781bd76eb --- /dev/null +++ b/tests/binary-patching.nix @@ -0,0 +1,15 @@ +{ version }: + +with import ./config.nix; + +mkDerivation { + name = "foo-${toString version}"; + builder = builtins.toFile "builder.sh" + '' + mkdir $out + seq 1 1000000 > $out/foo + ${if version == 2 then '' + echo bla >> $out/foo + '' else ""} + ''; +} diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh new file mode 100644 index 000000000..8d7788fb6 --- /dev/null +++ b/tests/binary-patching.sh @@ -0,0 +1,33 @@ +source common.sh + +mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches + +RESULT=$TEST_ROOT/result + +# Build version 1 and 2 of the "foo" package. +$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest1 \ + $($nixbuild -o $RESULT binary-patching.nix --arg version 1) + +out2=$($nixbuild -o $RESULT binary-patching.nix --arg version 2) +$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest2 $out2 +rm $RESULT + +# Generate a binary patch. +$NIX_BIN_DIR/generate-patches.pl $TEST_ROOT/cache2 $TEST_ROOT/patches \ + file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2 + +grep -q "patch {" $TEST_ROOT/manifest2 + +# Get rid of version 2. +$nixstore --delete $out2 +! test -e $out2 + +# Pull the manifest containing the patch. +clearManifests +$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest2 + +# To make sure that we're using the patch, delete the full NARs. +rm -f $TEST_ROOT/cache2/* + +# Now rebuild it. This should use the patch generated above. +$nixbuild -o $RESULT binary-patching.nix --arg version 2 diff --git a/tests/init.sh b/tests/init.sh index 64947031b..cdc7aca83 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -23,6 +23,8 @@ ln -s $nixinstantiate $NIX_BIN_DIR/ ln -s $nixhash $NIX_BIN_DIR/ ln -s $nixenv $NIX_BIN_DIR/ ln -s $nixworker $NIX_BIN_DIR/ +ln -s $TOP/src/bsdiff-*/bsdiff $NIX_BIN_DIR/ +ln -s $TOP/src/bsdiff-*/bspatch $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-prefetch-url $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-collect-garbage $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-build $NIX_BIN_DIR/ @@ -34,6 +36,7 @@ ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/nix/ ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/ +ln -s $TOP/scripts/generate-patches.pl $NIX_BIN_DIR/ ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/ cat > "$NIX_CONF_DIR"/nix.conf < $i.tmp \ -e "s^$REAL_BIN_DIR/nix-store^$NIX_BIN_DIR/nix-store^" \ diff --git a/tests/nix-push.sh b/tests/nix-push.sh index a9a2f5f9c..0a35e3b97 100644 --- a/tests/nix-push.sh +++ b/tests/nix-push.sh @@ -5,7 +5,7 @@ outPath=$($nixstore -r $drvPath) echo "pushing $drvPath" -mkdir $TEST_ROOT/cache +mkdir -p $TEST_ROOT/cache $NIX_BIN_DIR/nix-push \ --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath From f69626ed3eb9314bbdf9b0fe5497a0c3a3465d31 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 15:45:09 +0000 Subject: [PATCH 097/128] --- tests/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Makefile.am b/tests/Makefile.am index 88925fbd9..fb4a2285c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -32,5 +32,6 @@ EXTRA_DIST = $(TESTS) \ filter-source.nix \ export-graph.nix \ negative-caching.nix \ + binary-patching.nix \ $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) $(wildcard lang/*.flags) \ common.sh.in From 3d38a498404bf842ca479d42d18def1f472a6fb0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 17:32:25 +0000 Subject: [PATCH 098/128] * In the download size indication, take binary patches into account. Hopefully this doesn't slow things down too much. --- scripts/download-using-manifests.pl.in | 305 +++++++++++++------------ tests/binary-patching.nix | 2 +- tests/binary-patching.sh | 6 + 3 files changed, 168 insertions(+), 145 deletions(-) diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index 7b3f7ee66..7147090c4 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -31,6 +31,144 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") { } +sub isValidPath { + my $p = shift; + return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0; +} + + +sub parseHash { + my $hash = shift; + if ($hash =~ /^(.+):(.+)$/) { + return ($1, $2); + } else { + return ("md5", $hash); + } +} + + +# Compute the most efficient sequence of downloads to produce the +# given path. +sub computeSmallestDownload { + my $targetPath = shift; + + # Build a graph of all store paths that might contribute to the + # construction of $targetPath, and the special node "start". The + # edges are either patch operations, or downloads of full NAR + # files. The latter edges only occur between "start" and a store + # path. + my %graph; + + $graph{"start"} = {d => 0, pred => undef, edges => []}; + + my @queue = (); + my $queueFront = 0; + my %done; + + sub addNode { + my $graph = shift; + my $u = shift; + $$graph{$u} = {d => 999999999999, pred => undef, edges => []} + unless defined $$graph{$u}; + } + + sub addEdge { + my $graph = shift; + my $u = shift; + my $v = shift; + my $w = shift; + my $type = shift; + my $info = shift; + addNode $graph, $u; + push @{$$graph{$u}->{edges}}, + {weight => $w, start => $u, end => $v, type => $type, info => $info}; + my $n = scalar @{$$graph{$u}->{edges}}; + } + + push @queue, $targetPath; + + while ($queueFront < scalar @queue) { + my $u = $queue[$queueFront++]; + return if defined $done{$u}; + $done{$u} = 1; + + addNode \%graph, $u; + + # If the path already exists, it has distance 0 from the + # "start" node. + if (isValidPath($u)) { + addEdge \%graph, "start", $u, 0, "present", undef; + } + + else { + + # Add patch edges. + my $patchList = $patches{$u}; + foreach my $patch (@{$patchList}) { + if (isValidPath($patch->{basePath})) { + # !!! this should be cached + my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash}; + my $format = "--base32"; + $format = "" if $baseHashAlgo eq "md5"; + my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`; + chomp $hash; + next if $hash ne $baseHash; + } + push @queue, $patch->{basePath}; + addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch; + } + + # Add NAR file edges to the start node. + my $narFileList = $narFiles{$u}; + foreach my $narFile (@{$narFileList}) { + # !!! how to handle files whose size is not known in advance? + # For now, assume some arbitrary size (1 MB). + addEdge \%graph, "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile; + } + } + } + + + # Run Dijkstra's shortest path algorithm to determine the shortest + # sequence of download and/or patch actions that will produce + # $targetPath. + + my @todo = keys %graph; + + while (scalar @todo > 0) { + + # Remove the closest element from the todo list. + # !!! inefficient, use a priority queue + @todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo; + my $u = pop @todo; + + my $u_ = $graph{$u}; + + foreach my $edge (@{$u_->{edges}}) { + my $v_ = $graph{$edge->{end}}; + if ($v_->{d} > $u_->{d} + $edge->{weight}) { + $v_->{d} = $u_->{d} + $edge->{weight}; + # Store the edge; to edge->start is actually the + # predecessor. + $v_->{pred} = $edge; + } + } + } + + + # Retrieve the shortest path from "start" to $targetPath. + my @path = (); + my $cur = $targetPath; + return () unless defined $graph{$targetPath}->{pred}; + while ($cur ne "start") { + push @path, $graph{$cur}->{pred}; + $cur = $graph{$cur}->{pred}->{start}; + } + + return @path; +} + + # Parse the arguments. if ($ARGV[0] eq "--query") { @@ -46,6 +184,7 @@ if ($ARGV[0] eq "--query") { elsif ($cmd eq "info") { my $storePath = ; chomp $storePath; + my $info; if (defined $narFiles{$storePath}) { $info = @{$narFiles{$storePath}}[0]; @@ -57,13 +196,30 @@ if ($ARGV[0] eq "--query") { print "0\n"; next; # not an error } + print "1\n"; print "$info->{deriver}\n"; my @references = split " ", $info->{references}; print scalar @references, "\n"; print "$_\n" foreach @references; - my $size = $info->{size} || 0; - print "$size\n"; + + my @path = computeSmallestDownload $storePath; + + my $downloadSize = 0; + while (scalar @path > 0) { + my $edge = pop @path; + my $u = $edge->{start}; + my $v = $edge->{end}; + if ($edge->{type} eq "patch") { + $downloadSize += $edge->{info}->{size}; + } + elsif ($edge->{type} eq "narfile") { + $downloadSize += $edge->{info}->{size}; + } + } + + print "$downloadSize\n"; + my $narSize = $info->{narSize} || 0; print "$narSize\n"; } @@ -112,148 +268,9 @@ foreach my $localPath (@{$localPathList}) { } -# Build a graph of all store paths that might contribute to the -# construction of $targetPath, and the special node "start". The -# edges are either patch operations, or downloads of full NAR files. -# The latter edges only occur between "start" and a store path. - -my %graph; - -$graph{"start"} = {d => 0, pred => undef, edges => []}; - -my @queue = (); -my $queueFront = 0; -my %done; - -sub addToQueue { - my $v = shift; - return if defined $done{$v}; - $done{$v} = 1; - push @queue, $v; -} - -sub addNode { - my $u = shift; - $graph{$u} = {d => 999999999999, pred => undef, edges => []} - unless defined $graph{$u}; -} - -sub addEdge { - my $u = shift; - my $v = shift; - my $w = shift; - my $type = shift; - my $info = shift; - addNode $u; - push @{$graph{$u}->{edges}}, - {weight => $w, start => $u, end => $v, type => $type, info => $info}; - my $n = scalar @{$graph{$u}->{edges}}; -} - -addToQueue $targetPath; - -sub isValidPath { - my $p = shift; - return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0; -} - -sub parseHash { - my $hash = shift; - if ($hash =~ /^(.+):(.+)$/) { - return ($1, $2); - } else { - return ("md5", $hash); - } -} - -while ($queueFront < scalar @queue) { - my $u = $queue[$queueFront++]; -# print "$u\n"; - - addNode $u; - - # If the path already exists, it has distance 0 from the "start" - # node. - if (isValidPath($u)) { - addEdge "start", $u, 0, "present", undef; - } - - else { - - # Add patch edges. - my $patchList = $patches{$u}; - foreach my $patch (@{$patchList}) { - if (isValidPath($patch->{basePath})) { - # !!! this should be cached - my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash}; - my $format = "--base32"; - $format = "" if $baseHashAlgo eq "md5"; - my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`; - chomp $hash; - if ($hash ne $baseHash) { - print LOGFILE "$$ rejecting $patch->{basePath}\n"; - next; - } - } - addToQueue $patch->{basePath}; - addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch; - } - - # Add NAR file edges to the start node. - my $narFileList = $narFiles{$u}; - foreach my $narFile (@{$narFileList}) { - # !!! how to handle files whose size is not known in advance? - # For now, assume some arbitrary size (1 MB). - addEdge "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile; - if ($u eq $targetPath) { - my $size = $narFile->{size} || -1; - print LOGFILE "$$ full-download-would-be $size\n"; - } - } - - } -} - - -# Run Dijkstra's shortest path algorithm to determine the shortest -# sequence of download and/or patch actions that will produce -# $targetPath. - -sub byDistance { # sort by distance, reversed - return -($graph{$a}->{d} <=> $graph{$b}->{d}); -} - -my @todo = keys %graph; - -while (scalar @todo > 0) { - - # Remove the closest element from the todo list. - @todo = sort byDistance @todo; - my $u = pop @todo; - - my $u_ = $graph{$u}; - - foreach my $edge (@{$u_->{edges}}) { - my $v_ = $graph{$edge->{end}}; - if ($v_->{d} > $u_->{d} + $edge->{weight}) { - $v_->{d} = $u_->{d} + $edge->{weight}; - # Store the edge; to edge->start is actually the - # predecessor. - $v_->{pred} = $edge; - } - } -} - - -# Retrieve the shortest path from "start" to $targetPath. -my @path = (); -my $cur = $targetPath; -die "don't know how to produce $targetPath\n" - unless defined $graph{$targetPath}->{pred}; -while ($cur ne "start") { - push @path, $graph{$cur}->{pred}; - $cur = $graph{$cur}->{pred}->{start}; -} +# Compute the shortest path. +my @path = computeSmallestDownload $targetPath; +die "don't know how to produce $targetPath\n" if scalar @path == 0; # Traverse the shortest path, perform the actions described by the diff --git a/tests/binary-patching.nix b/tests/binary-patching.nix index 781bd76eb..afa0a0fb3 100644 --- a/tests/binary-patching.nix +++ b/tests/binary-patching.nix @@ -9,7 +9,7 @@ mkDerivation { mkdir $out seq 1 1000000 > $out/foo ${if version == 2 then '' - echo bla >> $out/foo + seq 1000000 1010000 >> $out/foo '' else ""} ''; } diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh index 8d7788fb6..26a499727 100644 --- a/tests/binary-patching.sh +++ b/tests/binary-patching.sh @@ -1,5 +1,7 @@ source common.sh +clearManifests + mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches RESULT=$TEST_ROOT/result @@ -29,5 +31,9 @@ $NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest2 # To make sure that we're using the patch, delete the full NARs. rm -f $TEST_ROOT/cache2/* +# Make sure that the download size prediction uses the patch rather +# than the full download. +$nixbuild -o $RESULT binary-patching.nix --arg version 2 --dry-run 2>&1 | grep -q "0.01 MiB" + # Now rebuild it. This should use the patch generated above. $nixbuild -o $RESULT binary-patching.nix --arg version 2 From a4f0365b2dfc95b60c42a702937fc801dcc8d81b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 17:41:59 +0000 Subject: [PATCH 099/128] * When checking whether a patch is applicable, for efficiency, use `nix-store -q --hash' to get the hash of the base path rather than `nix-hash'. However, only do this for estimating the size of a download, not for the actual substitution, because sometimes the contents of store paths are modified (which they shouldn't, of course). --- scripts/download-using-manifests.pl.in | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index 7147090c4..c31a294e8 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -51,6 +51,7 @@ sub parseHash { # given path. sub computeSmallestDownload { my $targetPath = shift; + my $fast = shift; # Build a graph of all store paths that might contribute to the # construction of $targetPath, and the special node "start". The @@ -110,8 +111,11 @@ sub computeSmallestDownload { my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash}; my $format = "--base32"; $format = "" if $baseHashAlgo eq "md5"; - my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`; + my $hash = $fast && $baseHashAlgo eq "sha256" + ? `$binDir/nix-store -q --hash "$patch->{basePath}"` + : `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`; chomp $hash; + $hash =~ s/.*://; next if $hash ne $baseHash; } push @queue, $patch->{basePath}; @@ -203,7 +207,7 @@ if ($ARGV[0] eq "--query") { print scalar @references, "\n"; print "$_\n" foreach @references; - my @path = computeSmallestDownload $storePath; + my @path = computeSmallestDownload $storePath, 1; my $downloadSize = 0; while (scalar @path > 0) { @@ -269,7 +273,7 @@ foreach my $localPath (@{$localPathList}) { # Compute the shortest path. -my @path = computeSmallestDownload $targetPath; +my @path = computeSmallestDownload $targetPath, 0; die "don't know how to produce $targetPath\n" if scalar @path == 0; From 1a211d812f1808a5a9128cbfd74bb59d07487b1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 17:54:49 +0000 Subject: [PATCH 100/128] * Oops. --- scripts/download-using-manifests.pl.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index c31a294e8..c50f540f3 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -90,7 +90,7 @@ sub computeSmallestDownload { while ($queueFront < scalar @queue) { my $u = $queue[$queueFront++]; - return if defined $done{$u}; + next if defined $done{$u}; $done{$u} = 1; addNode \%graph, $u; From d92ccbf1ac5ab68937a0e411f4daee7e427865bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 18:01:37 +0000 Subject: [PATCH 101/128] * Test whether sequences of patches work. --- tests/binary-patching.nix | 5 ++++- tests/binary-patching.sh | 47 +++++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/tests/binary-patching.nix b/tests/binary-patching.nix index afa0a0fb3..0de3028bd 100644 --- a/tests/binary-patching.nix +++ b/tests/binary-patching.nix @@ -8,8 +8,11 @@ mkDerivation { '' mkdir $out seq 1 1000000 > $out/foo - ${if version == 2 then '' + ${if version != 1 then '' seq 1000000 1010000 >> $out/foo '' else ""} + ${if version == 3 then '' + echo foobar >> $out/foo + '' else ""} ''; } diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh index 26a499727..c320dccc7 100644 --- a/tests/binary-patching.sh +++ b/tests/binary-patching.sh @@ -12,28 +12,47 @@ $NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest1 \ out2=$($nixbuild -o $RESULT binary-patching.nix --arg version 2) $NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest2 $out2 + +out3=$($nixbuild -o $RESULT binary-patching.nix --arg version 3) +$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest3 $out3 + rm $RESULT -# Generate a binary patch. +# Generate binary patches. $NIX_BIN_DIR/generate-patches.pl $TEST_ROOT/cache2 $TEST_ROOT/patches \ file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2 -grep -q "patch {" $TEST_ROOT/manifest2 +$NIX_BIN_DIR/generate-patches.pl $TEST_ROOT/cache2 $TEST_ROOT/patches \ + file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3 -# Get rid of version 2. -$nixstore --delete $out2 -! test -e $out2 +grep -q "patch {" $TEST_ROOT/manifest3 -# Pull the manifest containing the patch. +# Get rid of versions 2 and 3. +$nixstore --delete $out2 $out3 + +# Pull the manifest containing the patches. clearManifests -$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest2 +$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest3 -# To make sure that we're using the patch, delete the full NARs. -rm -f $TEST_ROOT/cache2/* - -# Make sure that the download size prediction uses the patch rather +# Make sure that the download size prediction uses the patches rather # than the full download. -$nixbuild -o $RESULT binary-patching.nix --arg version 2 --dry-run 2>&1 | grep -q "0.01 MiB" +$nixbuild -o $RESULT binary-patching.nix --arg version 3 --dry-run 2>&1 | grep -q "0.01 MiB" -# Now rebuild it. This should use the patch generated above. -$nixbuild -o $RESULT binary-patching.nix --arg version 2 +# Now rebuild it. This should use the two patches generated above. +rm -f $TEST_ROOT/var/log/nix/downloads +$nixbuild -o $RESULT binary-patching.nix --arg version 3 +rm $RESULT +[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ] + +# Add a patch from version 1 directly to version 3. +$NIX_BIN_DIR/generate-patches.pl $TEST_ROOT/cache2 $TEST_ROOT/patches \ + file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3 + +# Rebuild version 3. This should use the direct patch rather than the +# sequence of two patches. +$nixstore --delete $out2 $out3 +clearManifests +rm $TEST_ROOT/var/log/nix/downloads +$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest3 +$nixbuild -o $RESULT binary-patching.nix --arg version 3 +[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 1 ] From bd48fd97f646f068ae2725b5b2f048d45866119b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Nov 2010 09:15:45 +0000 Subject: [PATCH 102/128] * Impose a configurable time limit on patch generation. This is necessary because bsdiff has some pathological worst-case behaviour, e.g. O(n^2) on files consisting only of 0s. --- scripts/generate-patches.pl.in | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/generate-patches.pl.in b/scripts/generate-patches.pl.in index dba647350..3be30be00 100755 --- a/scripts/generate-patches.pl.in +++ b/scripts/generate-patches.pl.in @@ -15,6 +15,9 @@ $maxNarSize = 100 * 1024 * 1024 if !defined $maxNarSize; my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"}; $maxPatchFraction = 0.60 if !defined $maxPatchFraction; +my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"}; +$timeLimit = 180 if !defined $timeLimit; + die unless scalar @ARGV == 5; @@ -29,8 +32,6 @@ my $dstManifest = $ARGV[4]; my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; -print "TEMP = $tmpDir\n"; - #END { rmdir $tmpDir; } my %srcNarFiles; @@ -223,7 +224,7 @@ foreach my $p (keys %dstOutPaths) { foreach my $q (keys %srcOutPaths) { (my $name2, my $version2) = getNameVersion $q; - next unless defined $name2 && defined $version2; + next unless defined $name2 && defined $version2; if ($name eq $name2) { @@ -312,8 +313,13 @@ foreach my $p (keys %dstOutPaths) { next; } - system("@libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF") == 0 - or die "cannot compute binary diff"; + my $time1 = time(); + my $res = system("ulimit -t $timeLimit; @libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF"); + my $time2 = time(); + if ($res) { + warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n"; + next; + } my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die; chomp $baseHash; @@ -327,7 +333,7 @@ foreach my $p (keys %dstOutPaths) { my $narDiffSize = (stat "$tmpDir/DIFF")[7]; my $dstNarBz2Size = (stat $dstNarBz2)[7]; - print " size $narDiffSize; full size $dstNarBz2Size\n"; + print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n"; if ($narDiffSize >= $dstNarBz2Size) { print " rejecting; patch bigger than full archive\n"; From 9737a7eba0844e317591f16092879696c4f3feae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Nov 2010 09:26:05 +0000 Subject: [PATCH 103/128] * Don't generate patches if the system type differs. --- scripts/generate-patches.pl.in | 9 ++++++++- scripts/readmanifest.pm.in | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/generate-patches.pl.in b/scripts/generate-patches.pl.in index 3be30be00..05bf65093 100755 --- a/scripts/generate-patches.pl.in +++ b/scripts/generate-patches.pl.in @@ -228,6 +228,13 @@ foreach my $p (keys %dstOutPaths) { if ($name eq $name2) { + my $srcSystem = @{$dstNarFiles{$p}}[0]->{system}; + my $dstSystem = @{$srcNarFiles{$q}}[0]->{system}; + if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) { + print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n"; + next; + } + # If the sizes differ too much, then skip. This # disambiguates between, e.g., a real component and a # wrapper component (cf. Firefox in Nixpkgs). @@ -238,7 +245,7 @@ foreach my $p (keys %dstOutPaths) { # print " SIZE $srcSize $dstSize $ratio $q\n"; if ($ratio >= 3) { - print " SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n"; + print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n"; next; } diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index c2c4be89b..82a01e5ab 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -34,7 +34,7 @@ sub readManifest { my $manifestVersion = 2; my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); - my ($narHash, $narSize, $references, $deriver, $hashAlgo, $copyFrom); + my ($narHash, $narSize, $references, $deriver, $hashAlgo, $copyFrom, $system); while () { chomp; @@ -56,6 +56,7 @@ sub readManifest { undef $basePath; undef $baseHash; undef $patchType; + undef $system; $references = ""; $deriver = ""; $hashAlgo = "md5"; @@ -83,6 +84,7 @@ sub readManifest { , narHash => $narHash, narSize => $narSize , references => $references , deriver => $deriver, hashAlgo => $hashAlgo + , system => $system }; } @@ -128,6 +130,7 @@ sub readManifest { elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } + elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; } # Compatibility; elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } @@ -165,6 +168,7 @@ sub writeManifest { if defined $narFile->{references} && $narFile->{references} ne ""; print MANIFEST " Deriver: $narFile->{deriver}\n" if defined $narFile->{deriver} && $narFile->{deriver} ne ""; + print MANIFEST " System: $narFile->{system}\n" if defined $narFile->{system}; print MANIFEST "}\n"; } } From d3bba0c2d8b879950d55d508ef3fc8dec3559f8f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Nov 2010 15:26:28 +0000 Subject: [PATCH 104/128] * Move the patch generator into a module. --- scripts/GeneratePatches.pm.in | 335 +++++++++++++++++++++++++ scripts/Makefile.am | 6 +- scripts/generate-patches.pl.in | 429 --------------------------------- scripts/readmanifest.pm.in | 12 +- 4 files changed, 345 insertions(+), 437 deletions(-) create mode 100755 scripts/GeneratePatches.pm.in delete mode 100755 scripts/generate-patches.pl.in diff --git a/scripts/GeneratePatches.pm.in b/scripts/GeneratePatches.pm.in new file mode 100755 index 000000000..bb7111f8e --- /dev/null +++ b/scripts/GeneratePatches.pm.in @@ -0,0 +1,335 @@ +#! @perl@ -w -I@libexecdir@/nix + +use strict; +use File::Temp qw(tempdir); +use readmanifest; + + +# Some patch generations options. + +# Max size of NAR archives to generate patches for. +my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"}; +$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize; + +# If patch is bigger than this fraction of full archive, reject. +my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"}; +$maxPatchFraction = 0.60 if !defined $maxPatchFraction; + +my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"}; +$timeLimit = 180 if !defined $timeLimit; + +my $hashAlgo = "sha256"; + + +sub findOutputPaths { + my $narFiles = shift; + + my %outPaths; + + foreach my $p (keys %{$narFiles}) { + + # Ignore derivations. + next if ($p =~ /\.drv$/); + + # Ignore builders (too much ambiguity -- they're all called + # `builder.sh'). + next if ($p =~ /\.sh$/); + next if ($p =~ /\.patch$/); + + # Don't bother including tar files etc. + next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/); + + $outPaths{$p} = 1; + } + + return %outPaths; +} + + +sub getNameVersion { + my $p = shift; + $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; + my $name = $1; + my $version = $2; + return undef unless defined $name && defined $version; + $name =~ s/^-//; + $version =~ s/^-//; + return ($name, $version); +} + + +# A quick hack to get a measure of the `distance' between two +# versions: it's just the position of the first character that differs +# (or 999 if they are the same). +sub versionDiff { + my $s = shift; + my $t = shift; + my $i; + return 999 if $s eq $t; + for ($i = 0; $i < length $s; $i++) { + return $i if $i >= length $t or + substr($s, $i, 1) ne substr($t, $i, 1); + } + return $i; +} + + +sub getNarBz2 { + my $narPath = shift; + my $narFiles = shift; + my $storePath = shift; + + my $narFileList = $$narFiles{$storePath}; + die "missing path $storePath" unless defined $narFileList; + + my $narFile = @{$narFileList}[0]; + die unless defined $narFile; + + $narFile->{url} =~ /\/([^\/]+)$/; + die unless defined $1; + return "$narPath/$1"; +} + + +sub containsPatch { + my $patches = shift; + my $storePath = shift; + my $basePath = shift; + my $patchList = $$patches{$storePath}; + return 0 if !defined $patchList; + my $found = 0; + foreach my $patch (@{$patchList}) { + # !!! baseHash might differ + return 1 if $patch->{basePath} eq $basePath; + } + return 0; +} + + +sub generatePatches { + my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_; + + my %srcOutPaths = findOutputPaths $srcNarFiles; + my %dstOutPaths = findOutputPaths $dstNarFiles; + + # For each output path in the destination, see if we need to / can + # create a patch. + + print STDERR "creating patches...\n"; + + foreach my $p (keys %dstOutPaths) { + + # If exactly the same path already exists in the source, skip it. + next if defined $srcOutPaths{$p}; + + print " $p\n"; + + # If not, then we should find the paths in the source that are + # `most' likely to be present on a system that wants to + # install this path. + + (my $name, my $version) = getNameVersion $p; + next unless defined $name && defined $version; + + my @closest = (); + my $closestVersion; + my $minDist = -1; # actually, larger means closer + + # Find all source paths with the same name. + + foreach my $q (keys %srcOutPaths) { + (my $name2, my $version2) = getNameVersion $q; + next unless defined $name2 && defined $version2; + + if ($name eq $name2) { + + my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system}; + my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system}; + if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) { + print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n"; + next; + } + + # If the sizes differ too much, then skip. This + # disambiguates between, e.g., a real component and a + # wrapper component (cf. Firefox in Nixpkgs). + my $srcSize = @{$$srcNarFiles{$q}}[0]->{size}; + my $dstSize = @{$$dstNarFiles{$p}}[0]->{size}; + my $ratio = $srcSize / $dstSize; + $ratio = 1 / $ratio if $ratio < 1; + # print " SIZE $srcSize $dstSize $ratio $q\n"; + + if ($ratio >= 3) { + print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n"; + next; + } + + # If there are multiple matching names, include the + # ones with the closest version numbers. + my $dist = versionDiff $version, $version2; + if ($dist > $minDist) { + $minDist = $dist; + @closest = ($q); + $closestVersion = $version2; + } elsif ($dist == $minDist) { + push @closest, $q; + } + } + } + + if (scalar(@closest) == 0) { + print " NO BASE: $p\n"; + next; + } + + foreach my $closest (@closest) { + + # Generate a patch between $closest and $p. + print STDERR " $p <- $closest\n"; + + # If the patch already exists, skip it. + if (containsPatch($srcPatches, $p, $closest) || + containsPatch($dstPatches, $p, $closest)) + { + print " skipping, already exists\n"; + next; + } + + my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest; + my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p; + + if (! -f $srcNarBz2) { + warn "patch source archive $srcNarBz2 is missing\n"; + next; + } + + system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0 + or die "cannot unpack $srcNarBz2"; + + if ((stat "$tmpDir/A")[7] >= $maxNarSize) { + print " skipping, source is too large\n"; + next; + } + + system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0 + or die "cannot unpack $dstNarBz2"; + + if ((stat "$tmpDir/B")[7] >= $maxNarSize) { + print " skipping, destination is too large\n"; + next; + } + + my $time1 = time(); + my $res = system("ulimit -t $timeLimit; @libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF"); + my $time2 = time(); + if ($res) { + warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n"; + next; + } + + my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die; + chomp $baseHash; + + my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die; + chomp $narHash; + + my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die; + chomp $narDiffHash; + + my $narDiffSize = (stat "$tmpDir/DIFF")[7]; + my $dstNarBz2Size = (stat $dstNarBz2)[7]; + + print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n"; + + if ($narDiffSize >= $dstNarBz2Size) { + print " rejecting; patch bigger than full archive\n"; + next; + } + + if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) { + print " rejecting; patch too large relative to full archive\n"; + next; + } + + my $finalName = "$narDiffHash.nar-bsdiff"; + + if (-e "$patchesPath/$finalName") { + print " not copying, already exists\n"; + } + + else { + system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0 + or die "cannot copy diff"; + rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName") + or die "cannot rename $patchesPath/$finalName.tmp"; + } + + # Add the patch to the manifest. + addPatch $dstPatches, $p, + { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash" + , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash" + , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff" + }; + } + } +} + + +# Propagate useful patches from $srcPatches to $dstPatches. A patch +# is useful if it produces either paths in the $dstNarFiles or paths +# that can be used as the base for other useful patches. +sub propagatePatches { + my ($srcPatches, $dstNarFiles, $dstPatches) = @_; + + print STDERR "propagating patches...\n"; + + my $changed; + do { + # !!! we repeat this to reach the transitive closure; inefficient + $changed = 0; + + print STDERR "loop\n"; + + my %dstBasePaths; + foreach my $q (keys %{$dstPatches}) { + foreach my $patch (@{$$dstPatches{$q}}) { + $dstBasePaths{$patch->{basePath}} = 1; + } + } + + foreach my $p (keys %{$srcPatches}) { + my $patchList = $$srcPatches{$p}; + + my $include = 0; + + # Is path $p included in the destination? If so, include + # patches that produce it. + $include = 1 if defined $$dstNarFiles{$p}; + + # Is path $p a path that serves as a base for paths in the + # destination? If so, include patches that produce it. + # !!! check baseHash + $include = 1 if defined $dstBasePaths{$p}; + + if ($include) { + foreach my $patch (@{$patchList}) { + $changed = 1 if addPatch $dstPatches, $p, $patch; + } + } + + } + + } while $changed; +} + + +# Add all new patches in $srcPatches to $dstPatches. +sub copyPatches { + my ($srcPatches, $dstPatches) = @_; + foreach my $p (keys %{$srcPatches}) { + addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}}; + } +} + + +return 1; diff --git a/scripts/Makefile.am b/scripts/Makefile.am index aa5d6f78c..c55f92212 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -3,7 +3,7 @@ bin_SCRIPTS = nix-collect-garbage \ nix-install-package nix-channel nix-build \ nix-copy-closure -noinst_SCRIPTS = nix-profile.sh generate-patches.pl \ +noinst_SCRIPTS = nix-profile.sh GeneratePatches.pm \ find-runtime-roots.pl build-remote.pl nix-reduce-build \ copy-from-other-stores.pl nix-http-export.cgi @@ -17,7 +17,7 @@ install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other- $(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_DATA) ssh.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix - $(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix + $(INSTALL_PROGRAM) GeneratePatches.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters $(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters @@ -36,7 +36,7 @@ EXTRA_DIST = nix-collect-garbage.in \ nix-build.in \ download-using-manifests.pl.in \ copy-from-other-stores.pl.in \ - generate-patches.pl.in \ + GeneratePatches.pm.in \ nix-copy-closure.in \ find-runtime-roots.pl.in \ build-remote.pl.in \ diff --git a/scripts/generate-patches.pl.in b/scripts/generate-patches.pl.in deleted file mode 100755 index 05bf65093..000000000 --- a/scripts/generate-patches.pl.in +++ /dev/null @@ -1,429 +0,0 @@ -#! @perl@ -w -I@libexecdir@/nix - -use strict; -use File::Temp qw(tempdir); -use readmanifest; - - -# Some patch generations options. - -# Max size of NAR archives to generate patches for. -my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"}; -$maxNarSize = 100 * 1024 * 1024 if !defined $maxNarSize; - -# If patch is bigger than this fraction of full archive, reject. -my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"}; -$maxPatchFraction = 0.60 if !defined $maxPatchFraction; - -my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"}; -$timeLimit = 180 if !defined $timeLimit; - - -die unless scalar @ARGV == 5; - -my $hashAlgo = "sha256"; - -my $narDir = $ARGV[0]; -my $patchesDir = $ARGV[1]; -my $patchesURL = $ARGV[2]; -my $srcManifest = $ARGV[3]; -my $dstManifest = $ARGV[4]; - -my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1) - or die "cannot create a temporary directory"; - -#END { rmdir $tmpDir; } - -my %srcNarFiles; -my %srcLocalPaths; -my %srcPatches; - -my %dstNarFiles; -my %dstLocalPaths; -my %dstPatches; - -readManifest "$srcManifest", - \%srcNarFiles, \%srcLocalPaths, \%srcPatches; - -readManifest "$dstManifest", - \%dstNarFiles, \%dstLocalPaths, \%dstPatches; - - -sub findOutputPaths { - my $narFiles = shift; - - my %outPaths; - - foreach my $p (keys %{$narFiles}) { - - # Ignore derivations. - next if ($p =~ /\.drv$/); - - # Ignore builders (too much ambiguity -- they're all called - # `builder.sh'). - next if ($p =~ /\.sh$/); - next if ($p =~ /\.patch$/); - - # Don't bother including tar files etc. - next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/); - - $outPaths{$p} = 1; - } - - return %outPaths; -} - -print "finding src output paths...\n"; -my %srcOutPaths = findOutputPaths \%srcNarFiles; - -print "finding dst output paths...\n"; -my %dstOutPaths = findOutputPaths \%dstNarFiles; - - -sub getNameVersion { - my $p = shift; - $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; - my $name = $1; - my $version = $2; - return undef unless defined $name && defined $version; - $name =~ s/^-//; - $version =~ s/^-//; - return ($name, $version); -} - - -# A quick hack to get a measure of the `distance' between two -# versions: it's just the position of the first character that differs -# (or 999 if they are the same). -sub versionDiff { - my $s = shift; - my $t = shift; - my $i; - return 999 if $s eq $t; - for ($i = 0; $i < length $s; $i++) { - return $i if $i >= length $t or - substr($s, $i, 1) ne substr($t, $i, 1); - } - return $i; -} - - -sub getNarBz2 { - my $narFiles = shift; - my $storePath = shift; - - my $narFileList = $$narFiles{$storePath}; - die "missing path $storePath" unless defined $narFileList; - - my $narFile = @{$narFileList}[0]; - die unless defined $narFile; - - $narFile->{url} =~ /\/([^\/]+)$/; - die unless defined $1; - return "$narDir/$1"; -} - - -sub containsPatch { - my $patches = shift; - my $storePath = shift; - my $basePath = shift; - my $patchList = $$patches{$storePath}; - return 0 if !defined $patchList; - my $found = 0; - foreach my $patch (@{$patchList}) { - # !!! baseHash might differ - return 1 if $patch->{basePath} eq $basePath; - } - return 0; -} - - -# Compute the "weighted" number of uses of a path in the build graph. -sub computeUses { - my $narFiles = shift; - my $path = shift; - - # Find the deriver of $path. - return 1 unless defined $$narFiles{$path}; - my $deriver = @{$$narFiles{$path}}[0]->{deriver}; - return 1 unless defined $deriver && $deriver ne ""; - -# print " DERIVER $deriver\n"; - - # Optimisation: build the referrers graph from the references - # graph. - my %referrers; - foreach my $q (keys %{$narFiles}) { - my @refs = split " ", @{$$narFiles{$q}}[0]->{references}; - foreach my $r (@refs) { - $referrers{$r} = [] unless defined $referrers{$r}; - push @{$referrers{$r}}, $q; - } - } - - # Determine the shortest path from $deriver to all other reachable - # paths in the `referrers' graph. - - my %dist; - $dist{$deriver} = 0; - - my @queue = ($deriver); - my $pos = 0; - - while ($pos < scalar @queue) { - my $p = $queue[$pos]; - $pos++; - - foreach my $q (@{$referrers{$p}}) { - if (!defined $dist{$q}) { - $dist{$q} = $dist{$p} + 1; -# print " $q $dist{$q}\n"; - push @queue, $q; - } - } - } - - my $wuse = 1.0; - foreach my $user (keys %dist) { - next if $user eq $deriver; -# print " $user $dist{$user}\n"; - $wuse += 1.0 / 2.0**$dist{$user}; - } - -# print " XXX $path $wuse\n"; - - return $wuse; -} - - -# For each output path in the destination, see if we need to / can -# create a patch. - -print "creating patches...\n"; - -foreach my $p (keys %dstOutPaths) { - - # If exactly the same path already exists in the source, skip it. - next if defined $srcOutPaths{$p}; - - print " $p\n"; - - # If not, then we should find the paths in the source that are - # `most' likely to be present on a system that wants to install - # this path. - - (my $name, my $version) = getNameVersion $p; - next unless defined $name && defined $version; - - my @closest = (); - my $closestVersion; - my $minDist = -1; # actually, larger means closer - - # Find all source paths with the same name. - - foreach my $q (keys %srcOutPaths) { - (my $name2, my $version2) = getNameVersion $q; - next unless defined $name2 && defined $version2; - - if ($name eq $name2) { - - my $srcSystem = @{$dstNarFiles{$p}}[0]->{system}; - my $dstSystem = @{$srcNarFiles{$q}}[0]->{system}; - if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) { - print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n"; - next; - } - - # If the sizes differ too much, then skip. This - # disambiguates between, e.g., a real component and a - # wrapper component (cf. Firefox in Nixpkgs). - my $srcSize = @{$srcNarFiles{$q}}[0]->{size}; - my $dstSize = @{$dstNarFiles{$p}}[0]->{size}; - my $ratio = $srcSize / $dstSize; - $ratio = 1 / $ratio if $ratio < 1; -# print " SIZE $srcSize $dstSize $ratio $q\n"; - - if ($ratio >= 3) { - print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n"; - next; - } - - # If the numbers of weighted uses differ too much, then - # skip. This disambiguates between, e.g., the bootstrap - # GCC and the final GCC in Nixpkgs. -# my $srcUses = computeUses \%srcNarFiles, $q; -# my $dstUses = computeUses \%dstNarFiles, $p; -# $ratio = $srcUses / $dstUses; -# $ratio = 1 / $ratio if $ratio < 1; -# print " USE $srcUses $dstUses $ratio $q\n"; - -# if ($ratio >= 2) { -# print " SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n"; -# next; -# } - - # If there are multiple matching names, include the ones - # with the closest version numbers. - my $dist = versionDiff $version, $version2; - if ($dist > $minDist) { - $minDist = $dist; - @closest = ($q); - $closestVersion = $version2; - } elsif ($dist == $minDist) { - push @closest, $q; - } - } - } - - if (scalar(@closest) == 0) { - print " NO BASE: $p\n"; - next; - } - - foreach my $closest (@closest) { - - # Generate a patch between $closest and $p. - print " $p <- $closest\n"; - - # If the patch already exists, skip it. - if (containsPatch(\%srcPatches, $p, $closest) || - containsPatch(\%dstPatches, $p, $closest)) - { - print " skipping, already exists\n"; - next; - } - -# next; - - my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest; - my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p; - - if (! -f $srcNarBz2) { - warn "patch source archive $srcNarBz2 is missing\n"; - next; - } - - system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0 - or die "cannot unpack $srcNarBz2"; - - if ((stat "$tmpDir/A")[7] >= $maxNarSize) { - print " skipping, source is too large\n"; - next; - } - - system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0 - or die "cannot unpack $dstNarBz2"; - - if ((stat "$tmpDir/B")[7] >= $maxNarSize) { - print " skipping, destination is too large\n"; - next; - } - - my $time1 = time(); - my $res = system("ulimit -t $timeLimit; @libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF"); - my $time2 = time(); - if ($res) { - warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n"; - next; - } - - my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die; - chomp $baseHash; - - my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die; - chomp $narHash; - - my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die; - chomp $narDiffHash; - - my $narDiffSize = (stat "$tmpDir/DIFF")[7]; - my $dstNarBz2Size = (stat $dstNarBz2)[7]; - - print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n"; - - if ($narDiffSize >= $dstNarBz2Size) { - print " rejecting; patch bigger than full archive\n"; - next; - } - - if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) { - print " rejecting; patch too large relative to full archive\n"; - next; - } - - my $finalName = - "$narDiffHash.nar-bsdiff"; - - if (-e "$patchesDir/$finalName") { - print " not copying, already exists\n"; - } - - else { - - system("cp '$tmpDir/DIFF' '$patchesDir/$finalName.tmp'") == 0 - or die "cannot copy diff"; - - rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName") - or die "cannot rename $patchesDir/$finalName.tmp"; - - } - - # Add the patch to the manifest. - addPatch \%dstPatches, $p, - { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash" - , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash" - , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff" - }, 0; - } -} - - -# Add in any potentially useful patches in the source (namely, those -# patches that produce either paths in the destination or paths that -# can be used as the base for other useful patches). - -print "propagating patches...\n"; - -my $changed; -do { - # !!! we repeat this to reach the transitive closure; inefficient - $changed = 0; - - print "loop\n"; - - my %dstBasePaths; - foreach my $q (keys %dstPatches) { - foreach my $patch (@{$dstPatches{$q}}) { - $dstBasePaths{$patch->{basePath}} = 1; - } - } - - foreach my $p (keys %srcPatches) { - my $patchList = $srcPatches{$p}; - - my $include = 0; - - # Is path $p included in the destination? If so, include - # patches that produce it. - $include = 1 if defined $dstNarFiles{$p}; - - # Is path $p a path that serves as a base for paths in the - # destination? If so, include patches that produce it. - # !!! check baseHash - $include = 1 if defined $dstBasePaths{$p}; - - if ($include) { - foreach my $patch (@{$patchList}) { - $changed = 1 if addPatch \%dstPatches, $p, $patch; - } - } - - } - -} while $changed; - - -# Rewrite the manifest of the destination (with the new patches). -writeManifest "${dstManifest}", - \%dstNarFiles, \%dstPatches; diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index 82a01e5ab..be0dda616 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -146,7 +146,7 @@ sub readManifest { sub writeManifest { - my ($manifest, $narFiles, $patches) = @_; + my ($manifest, $narFiles, $patches, $noCompress) = @_; open MANIFEST, ">$manifest.tmp"; # !!! check exclusive @@ -198,11 +198,13 @@ sub writeManifest { # Create a bzipped manifest. - system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0 - or die "cannot compress manifest"; + unless (defined $noCompress) { + system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0 + or die "cannot compress manifest"; - rename("$manifest.bz2.tmp", "$manifest.bz2") - or die "cannot rename $manifest.bz2.tmp: $!"; + rename("$manifest.bz2.tmp", "$manifest.bz2") + or die "cannot rename $manifest.bz2.tmp: $!"; + } } From 77f7a6d591e32a4a475552f3e67e3e67b7f71a10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Nov 2010 17:07:04 +0000 Subject: [PATCH 105/128] * Quick hack around one of the pathological cases in bsdiff (namely, when there is a long region of 0s). On one 86 MiB case, this cut patch generation time from 44m to 2m. --- src/bsdiff-4.3/bsdiff.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c index 150a7f79c..374ed038f 100644 --- a/src/bsdiff-4.3/bsdiff.c +++ b/src/bsdiff-4.3/bsdiff.c @@ -277,6 +277,7 @@ int main(int argc,char *argv[]) for(scsc=scan+=len;scan 64 * 1024) break; for(;scsc Date: Sun, 5 Dec 2010 17:36:02 +0000 Subject: [PATCH 106/128] * Add a script `nix-generate-patches'. * Fix the binary patching test. --- scripts/Makefile.am | 5 ++-- scripts/nix-generate-patches.in | 42 +++++++++++++++++++++++++++++++++ tests/binary-patching.sh | 6 ++--- tests/init.sh | 6 +++-- 4 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 scripts/nix-generate-patches.in diff --git a/scripts/Makefile.am b/scripts/Makefile.am index c55f92212..5f20dcfaf 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -1,7 +1,7 @@ bin_SCRIPTS = nix-collect-garbage \ nix-pull nix-push nix-prefetch-url \ nix-install-package nix-channel nix-build \ - nix-copy-closure + nix-copy-closure nix-generate-patches noinst_SCRIPTS = nix-profile.sh GeneratePatches.pm \ find-runtime-roots.pl build-remote.pl nix-reduce-build \ @@ -41,4 +41,5 @@ EXTRA_DIST = nix-collect-garbage.in \ find-runtime-roots.pl.in \ build-remote.pl.in \ nix-reduce-build.in \ - nix-http-export.cgi.in + nix-http-export.cgi.in \ + nix-generate-patches.in diff --git a/scripts/nix-generate-patches.in b/scripts/nix-generate-patches.in new file mode 100644 index 000000000..d22098a10 --- /dev/null +++ b/scripts/nix-generate-patches.in @@ -0,0 +1,42 @@ +#! @perl@ -w -I@libexecdir@/nix + +use strict; +use File::Temp qw(tempdir); +use readmanifest; +use GeneratePatches; + +if (scalar @ARGV != 5) { + print STDERR < 1, TMPDIR => 1) + or die "cannot create a temporary directory"; + +generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches, + $narPath, $patchesPath, $patchesURL, $tmpDir; + +propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches; + +writeManifest $dstManifest, \%dstNarFiles, \%dstPatches; diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh index c320dccc7..60e57b4b0 100644 --- a/tests/binary-patching.sh +++ b/tests/binary-patching.sh @@ -19,10 +19,10 @@ $NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest3 $out3 rm $RESULT # Generate binary patches. -$NIX_BIN_DIR/generate-patches.pl $TEST_ROOT/cache2 $TEST_ROOT/patches \ +$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2 -$NIX_BIN_DIR/generate-patches.pl $TEST_ROOT/cache2 $TEST_ROOT/patches \ +$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3 grep -q "patch {" $TEST_ROOT/manifest3 @@ -45,7 +45,7 @@ rm $RESULT [ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ] # Add a patch from version 1 directly to version 3. -$NIX_BIN_DIR/generate-patches.pl $TEST_ROOT/cache2 $TEST_ROOT/patches \ +$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3 # Rebuild version 3. This should use the direct patch rather than the diff --git a/tests/init.sh b/tests/init.sh index cdc7aca83..719d12d6f 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -31,12 +31,13 @@ ln -s $TOP/scripts/nix-build $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-install-package $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-push $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-pull $NIX_BIN_DIR/ +ln -s $TOP/scripts/nix-generate-patches $NIX_BIN_DIR/ mkdir $NIX_BIN_DIR/nix ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/nix/ ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/ -ln -s $TOP/scripts/generate-patches.pl $NIX_BIN_DIR/ +ln -s $TOP/scripts/GeneratePatches.pm $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/ cat > "$NIX_CONF_DIR"/nix.conf < $i.tmp \ -e "s^$REAL_BIN_DIR/nix-store^$NIX_BIN_DIR/nix-store^" \ From 365f3028ddfb5487f35ebbb9adc42ddf9459113d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 5 Dec 2010 17:50:29 +0000 Subject: [PATCH 107/128] * Use CamelCase for the Perl modules. --- scripts/GeneratePatches.pm.in | 1 - scripts/Makefile.am | 20 +++++++++---------- scripts/{readconfig.pm.in => NixConfig.pm.in} | 0 .../{readmanifest.pm.in => NixManifest.pm.in} | 0 scripts/{ssh.pm => SSH.pm} | 0 scripts/build-remote.pl.in | 2 +- scripts/download-using-manifests.pl.in | 2 +- scripts/nix-copy-closure.in | 2 +- scripts/nix-generate-patches.in | 2 +- scripts/nix-pull.in | 2 +- scripts/nix-push.in | 2 +- tests/init.sh | 4 ++-- 12 files changed, 18 insertions(+), 19 deletions(-) rename scripts/{readconfig.pm.in => NixConfig.pm.in} (100%) rename scripts/{readmanifest.pm.in => NixManifest.pm.in} (100%) rename scripts/{ssh.pm => SSH.pm} (100%) diff --git a/scripts/GeneratePatches.pm.in b/scripts/GeneratePatches.pm.in index bb7111f8e..2d2653255 100755 --- a/scripts/GeneratePatches.pm.in +++ b/scripts/GeneratePatches.pm.in @@ -2,7 +2,6 @@ use strict; use File::Temp qw(tempdir); -use readmanifest; # Some patch generations options. diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 5f20dcfaf..60bb0a9b8 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -7,17 +7,17 @@ noinst_SCRIPTS = nix-profile.sh GeneratePatches.pm \ find-runtime-roots.pl build-remote.pl nix-reduce-build \ copy-from-other-stores.pl nix-http-export.cgi -nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl +nix-pull nix-push: NixManifest.pm NixConfig.pm download-using-manifests.pl -install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl +install-exec-local: NixManifest.pm GeneratePatches.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl $(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d $(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix - $(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix - $(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix - $(INSTALL_DATA) ssh.pm $(DESTDIR)$(libexecdir)/nix + $(INSTALL_DATA) NixManifest.pm $(DESTDIR)$(libexecdir)/nix + $(INSTALL_DATA) NixConfig.pm $(DESTDIR)$(libexecdir)/nix + $(INSTALL_DATA) SSH.pm $(DESTDIR)$(libexecdir)/nix + $(INSTALL_DATA) GeneratePatches.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix - $(INSTALL_PROGRAM) GeneratePatches.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters $(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters @@ -30,13 +30,13 @@ EXTRA_DIST = nix-collect-garbage.in \ nix-pull.in nix-push.in nix-profile.sh.in \ nix-prefetch-url.in nix-install-package.in \ nix-channel.in \ - readmanifest.pm.in \ - readconfig.pm.in \ - ssh.pm \ + NixManifest.pm.in \ + NixConfig.pm.in \ + SSH.pm \ + GeneratePatches.pm.in \ nix-build.in \ download-using-manifests.pl.in \ copy-from-other-stores.pl.in \ - GeneratePatches.pm.in \ nix-copy-closure.in \ find-runtime-roots.pl.in \ build-remote.pl.in \ diff --git a/scripts/readconfig.pm.in b/scripts/NixConfig.pm.in similarity index 100% rename from scripts/readconfig.pm.in rename to scripts/NixConfig.pm.in diff --git a/scripts/readmanifest.pm.in b/scripts/NixManifest.pm.in similarity index 100% rename from scripts/readmanifest.pm.in rename to scripts/NixManifest.pm.in diff --git a/scripts/ssh.pm b/scripts/SSH.pm similarity index 100% rename from scripts/ssh.pm rename to scripts/SSH.pm diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index f9bff9c41..c551f6360 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -3,7 +3,7 @@ use Fcntl ':flock'; use English '-no_match_vars'; use IO::Handle; -use ssh qw/sshOpts openSSHConnection/; +use SSH qw/sshOpts openSSHConnection/; no warnings('once'); diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index c50f540f3..47ff09e60 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -1,7 +1,7 @@ #! @perl@ -w -I@libexecdir@/nix use strict; -use readmanifest; +use NixManifest; use POSIX qw(strftime); use File::Temp qw(tempdir); diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index a477cc01a..1f1fded36 100644 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -1,6 +1,6 @@ #! @perl@ -w -I@libexecdir@/nix -use ssh; +use SSH; my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; diff --git a/scripts/nix-generate-patches.in b/scripts/nix-generate-patches.in index d22098a10..c96cc704a 100644 --- a/scripts/nix-generate-patches.in +++ b/scripts/nix-generate-patches.in @@ -2,7 +2,7 @@ use strict; use File::Temp qw(tempdir); -use readmanifest; +use NixManifest; use GeneratePatches; if (scalar @ARGV != 5) { diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index e2a0cc1fa..f3b533ff7 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -2,7 +2,7 @@ use strict; use File::Temp qw(tempdir); -use readmanifest; +use NixManifest; my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; diff --git a/scripts/nix-push.in b/scripts/nix-push.in index 6760baa7e..1d8ba86a8 100644 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -2,7 +2,7 @@ use strict; use File::Temp qw(tempdir); -use readmanifest; +use NixManifest; my $hashAlgo = "sha256"; diff --git a/tests/init.sh b/tests/init.sh index 719d12d6f..104da432e 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -38,7 +38,7 @@ ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/GeneratePatches.pm $NIX_BIN_DIR/nix/ -ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/ +ln -s $TOP/scripts/NixManifest.pm $NIX_BIN_DIR/nix/ cat > "$NIX_CONF_DIR"/nix.conf < $i.tmp \ From de79d23f76c13da820f7167cd980264b2de28bd2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 5 Dec 2010 18:23:19 +0000 Subject: [PATCH 108/128] * Retry a transaction if SQLite returns SQLITE_BUSY. This can happen even with a very long busy timeout, because SQLITE_BUSY is also returned to resolve deadlocks. This should get rid of random "database is locked" errors. This is kind of hard to test though. * Fix a horrible bug in deleteFromStore(): deletePathWrapped() should be called after committing the transaction, not before, because the commit might not succeed. --- src/libstore/local-store.cc | 144 +++++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 61 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 37bbbfdad..88548706b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -22,22 +22,31 @@ namespace nix { - -class SQLiteError : public Error + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + + +static void throwSQLiteError(sqlite3 * db, const format & f) + __attribute__ ((noreturn)); + +static void throwSQLiteError(sqlite3 * db, const format & f) { -public: - SQLiteError(sqlite3 * db, const format & f) - : Error(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)) - { + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY) { + printMsg(lvlError, "warning: SQLite database is busy"); + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); } -}; + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} SQLite::~SQLite() { try { if (db && sqlite3_close(db) != SQLITE_OK) - throw SQLiteError(db, "closing database"); + throwSQLiteError(db, "closing database"); } catch (...) { ignoreException(); } @@ -49,7 +58,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) checkInterrupt(); assert(!stmt); if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); + throwSQLiteError(db, "creating statement"); this->db = db; } @@ -58,7 +67,7 @@ void SQLiteStmt::reset() { assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) - throw SQLiteError(db, "resetting statement"); + throwSQLiteError(db, "resetting statement"); curArg = 1; } @@ -67,7 +76,7 @@ SQLiteStmt::~SQLiteStmt() { try { if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); + throwSQLiteError(db, "finalizing statement"); } catch (...) { ignoreException(); } @@ -77,28 +86,28 @@ SQLiteStmt::~SQLiteStmt() void SQLiteStmt::bind(const string & value) { if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } void SQLiteStmt::bind(int value) { if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } void SQLiteStmt::bind64(long long value) { if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } void SQLiteStmt::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } @@ -132,14 +141,14 @@ struct SQLiteTxn SQLiteTxn(sqlite3 * db) : active(false) { this->db = db; if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "starting transaction"); + throwSQLiteError(db, "starting transaction"); active = true; } void commit() { if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "committing transaction"); + throwSQLiteError(db, "committing transaction"); active = false; } @@ -147,7 +156,7 @@ struct SQLiteTxn { try { if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "aborting transaction"); + throwSQLiteError(db, "aborting transaction"); } catch (...) { ignoreException(); } @@ -289,10 +298,10 @@ void LocalStore::openDB(bool create) throw Error("cannot open SQLite database"); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throw SQLiteError(db, "setting timeout"); + throwSQLiteError(db, "setting timeout"); if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "enabling foreign keys"); + throwSQLiteError(db, "enabling foreign keys"); /* !!! check whether sqlite has been built with foreign key support */ @@ -303,7 +312,7 @@ void LocalStore::openDB(bool create) crashes. */ string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting synchronous mode"); + throwSQLiteError(db, "setting synchronous mode"); /* Set the SQLite journal mode. WAL mode is fastest, but doesn't seem entirely stable at the moment (Oct. 2010). Thus, use @@ -314,31 +323,31 @@ void LocalStore::openDB(bool create) SQLiteStmt stmt; stmt.create(db, "pragma main.journal_mode;"); if (sqlite3_step(stmt) != SQLITE_ROW) - throw SQLiteError(db, "querying journal mode"); + throwSQLiteError(db, "querying journal mode"); prevMode = string((const char *) sqlite3_column_text(stmt, 0)); } if (prevMode != mode && sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting journal mode"); + throwSQLiteError(db, "setting journal mode"); /* Increase the auto-checkpoint interval to 8192 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting autocheckpoint interval"); + throwSQLiteError(db, "setting autocheckpoint interval"); /* Initialise the database schema, if necessary. */ if (create) { #include "schema.sql.hh" if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "initialising database schema"); + throwSQLiteError(db, "initialising database schema"); } /* Backwards compatibility with old (pre-release) databases. Can remove this eventually. */ if (sqlite3_table_column_metadata(db, 0, "ValidPaths", "narSize", 0, 0, 0, 0, 0) != SQLITE_OK) { if (sqlite3_exec(db, "alter table ValidPaths add column narSize integer" , 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "adding column narSize"); + throwSQLiteError(db, "adding column narSize"); } /* Prepare SQL statements. */ @@ -460,7 +469,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) else stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); + throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); unsigned long long id = sqlite3_last_insert_rowid(db); /* If this is a derivation, then store the derivation outputs in @@ -475,7 +484,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) stmtAddDerivationOutput.bind(i->first); stmtAddDerivationOutput.bind(i->second.path); if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throw SQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); } } @@ -489,7 +498,7 @@ void LocalStore::addReference(unsigned long long referrer, unsigned long long re stmtAddReference.bind(referrer); stmtAddReference.bind(reference); if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throw SQLiteError(db, "adding reference to database"); + throwSQLiteError(db, "adding reference to database"); } @@ -498,15 +507,23 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) assert(info.hash.type == htSHA256); ValidPathInfo info2(info); if (info2.registrationTime == 0) info2.registrationTime = time(0); - - SQLiteTxn txn(db); - - unsigned long long id = addValidPath(info2); - foreach (PathSet::const_iterator, i, info2.references) - addReference(id, queryValidPathId(*i)); - - txn.commit(); + while (1) { + try { + SQLiteTxn txn(db); + + unsigned long long id = addValidPath(info2); + + foreach (PathSet::const_iterator, i, info2.references) + addReference(id, queryValidPathId(*i)); + + txn.commit(); + break; + } catch (SQLiteBusy & e) { + /* Retry; the `txn' destructor will roll back the current + transaction. */ + } + } } @@ -517,7 +534,7 @@ void LocalStore::registerFailedPath(const Path & path) stmtRegisterFailedPath.bind(path); stmtRegisterFailedPath.bind(time(0)); if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throw SQLiteError(db, format("registering failed path `%1%'") % path); + throwSQLiteError(db, format("registering failed path `%1%'") % path); } @@ -527,7 +544,7 @@ bool LocalStore::hasPathFailed(const Path & path) stmtHasPathFailed.bind(path); int res = sqlite3_step(stmtHasPathFailed); if (res != SQLITE_DONE && res != SQLITE_ROW) - throw SQLiteError(db, "querying whether path failed"); + throwSQLiteError(db, "querying whether path failed"); return res == SQLITE_ROW; } @@ -545,7 +562,7 @@ PathSet LocalStore::queryFailedPaths() } if (r != SQLITE_DONE) - throw SQLiteError(db, "error querying failed paths"); + throwSQLiteError(db, "error querying failed paths"); return res; } @@ -559,7 +576,7 @@ void LocalStore::clearFailedPaths(const PathSet & paths) SQLiteStmtUse use(stmtClearFailedPath); stmtClearFailedPath.bind(*i); if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throw SQLiteError(db, format("clearing failed path `%1%' in database") % *i); + throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); } txn.commit(); @@ -594,7 +611,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) int r = sqlite3_step(stmtQueryPathInfo); if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); + if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); info.id = sqlite3_column_int(stmtQueryPathInfo, 0); @@ -622,7 +639,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting references of `%1%'") % path); + throwSQLiteError(db, format("error getting references of `%1%'") % path); return info; } @@ -635,7 +652,7 @@ unsigned long long LocalStore::queryValidPathId(const Path & path) int res = sqlite3_step(stmtQueryPathInfo); if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - throw SQLiteError(db, "querying path in database"); + throwSQLiteError(db, "querying path in database"); } @@ -645,7 +662,7 @@ bool LocalStore::isValidPath(const Path & path) stmtQueryPathInfo.bind(path); int res = sqlite3_step(stmtQueryPathInfo); if (res != SQLITE_DONE && res != SQLITE_ROW) - throw SQLiteError(db, "querying path in database"); + throwSQLiteError(db, "querying path in database"); return res == SQLITE_ROW; } @@ -665,7 +682,7 @@ PathSet LocalStore::queryValidPaths() } if (r != SQLITE_DONE) - throw SQLiteError(db, "error getting valid paths"); + throwSQLiteError(db, "error getting valid paths"); return res; } @@ -695,7 +712,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting references of `%1%'") % path); + throwSQLiteError(db, format("error getting references of `%1%'") % path); } @@ -721,7 +738,7 @@ PathSet LocalStore::queryValidDerivers(const Path & path) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting valid derivers of `%1%'") % path); + throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); return derivers; } @@ -743,7 +760,7 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting outputs of `%1%'") % path); + throwSQLiteError(db, format("error getting outputs of `%1%'") % path); return outputs; } @@ -890,7 +907,7 @@ void LocalStore::invalidatePath(const Path & path) stmtInvalidatePath.bind(path); if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throw SQLiteError(db, format("invalidating path `%1%' in database") % path); + throwSQLiteError(db, format("invalidating path `%1%' in database") % path); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ @@ -1248,20 +1265,25 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr assertStorePath(path); - SQLiteTxn txn(db); + while (1) { + try { + SQLiteTxn txn(db); - if (isValidPath(path)) { - PathSet referrers; queryReferrers(path, referrers); - referrers.erase(path); /* ignore self-references */ - if (!referrers.empty()) - throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") - % path % showPaths(referrers)); - invalidatePath(path); + if (isValidPath(path)) { + PathSet referrers; queryReferrers(path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) + throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") + % path % showPaths(referrers)); + invalidatePath(path); + } + + txn.commit(); + break; + } catch (SQLiteBusy & e) { }; } - + deletePathWrapped(path, bytesFreed, blocksFreed); - - txn.commit(); } From 8062d3af30b27eb4d617c14856d4f3a173f8012e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 6 Dec 2010 15:29:38 +0000 Subject: [PATCH 109/128] * `nix-store --verify --check-contents': don't hold the global GC lock while checking the contents, since this operation can take a very long time to finish. Also, fill in missing narSize fields in the DB while doing this. --- src/libstore/local-store.cc | 55 ++++++++++++++++++++++++++++++------- src/libstore/local-store.hh | 3 ++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 88548706b..3782d1b16 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -353,6 +353,8 @@ void LocalStore::openDB(bool create) /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); + stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ? where path = ?;"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -645,6 +647,21 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } +/* Update path info in the database. Currently only updated the + narSize field. */ +void LocalStore::updatePathInfo(const ValidPathInfo & info) +{ + SQLiteStmtUse use(stmtUpdatePathInfo); + if (info.narSize != 0) + stmtUpdatePathInfo.bind64(info.narSize); + else + stmtUpdatePathInfo.bind(); // null + stmtUpdatePathInfo.bind(info.path); + if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) + throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); +} + + unsigned long long LocalStore::queryValidPathId(const Path & path) { SQLiteStmtUse use(stmtQueryPathInfo); @@ -1305,23 +1322,41 @@ void LocalStore::verifyStore(bool checkContents) foreach (PathSet::iterator, i, validPaths2) verifyPath(*i, store, done, validPaths); + /* Release the GC lock so that checking content hashes (which can + take ages) doesn't block the GC or builds. */ + fdGCLock.close(); + /* Optionally, check the content hashes (slow). */ if (checkContents) { printMsg(lvlInfo, "checking hashes..."); foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfo(*i); + try { + ValidPathInfo info = queryPathInfo(*i); - /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); - Hash current = hashPath(info.hash.type, *i).first; - if (current != info.hash) { - printMsg(lvlError, format("path `%1%' was modified! " - "expected hash `%2%', got `%3%'") - % *i % printHash(info.hash) % printHash(current)); - } + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); + HashResult current = hashPath(info.hash.type, *i); + + if (current.first != info.hash) { + printMsg(lvlError, format("path `%1%' was modified! " + "expected hash `%2%', got `%3%'") + % *i % printHash(info.hash) % printHash(current.first)); + } else { + /* Fill in missing narSize fields (from old stores). */ + if (info.narSize == 0) { + printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); + info.narSize = current.second; + updatePathInfo(info); + } + } - /* !!! Check info.narSize */ + } catch (Error & e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(*i)) throw; + printMsg(lvlError, format("warning: %1%") % e.msg()); + } } } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4076e5957..f270fb723 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -203,6 +203,7 @@ private: /* Some precompiled SQLite statements. */ SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; SQLiteStmt stmtAddReference; SQLiteStmt stmtQueryPathInfo; SQLiteStmt stmtQueryReferences; @@ -235,6 +236,8 @@ private: void verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths); + void updatePathInfo(const ValidPathInfo & info); + void upgradeStore6(); PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); From 7d0444e2446c71e79e49f46f371c62f6d20488d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Dec 2010 12:33:42 +0000 Subject: [PATCH 110/128] * Bad things happen when a filehandle has the same name as a module. --- scripts/SSH.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/SSH.pm b/scripts/SSH.pm index 44a0e6f31..3b546ca52 100644 --- a/scripts/SSH.pm +++ b/scripts/SSH.pm @@ -24,9 +24,9 @@ sub openSSHConnection { # child continues to run if we are killed. So instead make SSH # print "started" when it has established the connection, and wait # until we see that. - open SSH, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die; + open SSHPIPE, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die; - while () { + while () { chomp; if ($_ eq "started") { $sshStarted = 1; From e4720b1a79f549b3bcb0e33fe999d02d12e3ed23 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Dec 2010 18:19:15 +0000 Subject: [PATCH 111/128] * Ignore the result of sqlite3_reset(). --- src/libstore/local-store.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3782d1b16..b80f66412 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,9 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) void SQLiteStmt::reset() { assert(stmt); - if (sqlite3_reset(stmt) != SQLITE_OK) - throwSQLiteError(db, "resetting statement"); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); curArg = 1; } @@ -333,7 +334,7 @@ void LocalStore::openDB(bool create) /* Increase the auto-checkpoint interval to 8192 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ - if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "setting autocheckpoint interval"); /* Initialise the database schema, if necessary. */ From 4d5777681309947041ecc297074f4ce537547553 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Dec 2010 11:45:56 +0000 Subject: [PATCH 112/128] * Use SQLite 3.7.4. --- configure.ac | 8 ++++---- externals/Makefile.am | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index c2599a75f..2a8bedb0d 100644 --- a/configure.ac +++ b/configure.ac @@ -247,12 +247,12 @@ AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], [prefix of SQLite]), sqlite=$withval, sqlite=) AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") -SQLITE_VERSION=3.7.2 +SQLITE_VERSION=3070400 AC_SUBST(SQLITE_VERSION) if test -z "$sqlite"; then - sqlite_lib='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)/libsqlite3.la' - sqlite_include='-I${top_builddir}/externals/sqlite-$(SQLITE_VERSION)' - sqlite_bin='${top_builddir}/externals/sqlite-$(SQLITE_VERSION)' + sqlite_lib='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)/libsqlite3.la' + sqlite_include='-I${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)' + sqlite_bin='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)' else sqlite_lib="-L$sqlite/lib -lsqlite3" sqlite_include="-I$sqlite/include" diff --git a/externals/Makefile.am b/externals/Makefile.am index bdc29fcb5..563f5424d 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -29,8 +29,8 @@ endif # SQLite -SQLITE = sqlite-$(SQLITE_VERSION) -SQLITE_TAR = sqlite-amalgamation-$(SQLITE_VERSION).tar.gz +SQLITE = sqlite-autoconf-$(SQLITE_VERSION) +SQLITE_TAR = sqlite-autoconf-$(SQLITE_VERSION).tar.gz $(SQLITE_TAR): @echo "Nix requires the SQLite library to build." From 542fc6906297fcb72cbcb01d688c2f27819e0cf7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 08:39:10 +0000 Subject: [PATCH 113/128] * When doing a query (e.g. `nix-store -r --dry-run'), don't make a lot of expensive calls to `nix-store --check-validity'. --- scripts/download-using-manifests.pl.in | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index 47ff09e60..29321bff9 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -12,6 +12,10 @@ STDOUT->autoflush(1); my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests"); my $logFile = "@localstatedir@/log/nix/downloads"; +# For queries, skip expensive calls to nix-hash etc. We're just +# estimating the expected download size. +my $fast = 1; + # Load all manifests. my %narFiles; @@ -33,7 +37,11 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") { sub isValidPath { my $p = shift; - return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0; + if ($fast) { + return -e $p; + } else { + return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0; + } } @@ -51,7 +59,6 @@ sub parseHash { # given path. sub computeSmallestDownload { my $targetPath = shift; - my $fast = shift; # Build a graph of all store paths that might contribute to the # construction of $targetPath, and the special node "start". The @@ -207,7 +214,7 @@ if ($ARGV[0] eq "--query") { print scalar @references, "\n"; print "$_\n" foreach @references; - my @path = computeSmallestDownload $storePath, 1; + my @path = computeSmallestDownload $storePath; my $downloadSize = 0; while (scalar @path > 0) { @@ -241,6 +248,7 @@ elsif ($ARGV[0] ne "--substitute") { die unless scalar @ARGV == 2; my $targetPath = $ARGV[1]; +$fast = 0; # Create a temporary directory. @@ -273,7 +281,7 @@ foreach my $localPath (@{$localPathList}) { # Compute the shortest path. -my @path = computeSmallestDownload $targetPath, 0; +my @path = computeSmallestDownload $targetPath; die "don't know how to produce $targetPath\n" if scalar @path == 0; From d7ca6f44eb506ec315db2ba59216de59de2758b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 13:19:46 +0000 Subject: [PATCH 114/128] * Update some comments. --- src/libstore/build.cc | 2 +- src/nix-worker/nix-worker.cc | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8b8be3e80..0be9c42e0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1715,7 +1715,7 @@ void DerivationGoal::startBuilder() case 0: /* Warning: in the child we should absolutely not make any - Berkeley DB calls! */ + SQLite calls! */ try { /* child */ diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 081b5b2fd..ec5ca58aa 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -712,9 +712,8 @@ static void daemonLoop() while (1) { try { - /* Important: the server process *cannot* open the - Berkeley DB environment, because it doesn't like forks - very much. */ + /* Important: the server process *cannot* open the SQLite + database, because it doesn't like forks very much. */ assert(!store); /* Accept a connection. */ From 100becf8d1b95055c73a024f1a2b318690de72f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 13:32:58 +0000 Subject: [PATCH 115/128] * createDirs(path): if path already exists, make sure it's a directory. * Provide a C++ wrapper around lstat(). --- src/libutil/util.cc | 42 ++++++++++++++++++++++-------------------- src/libutil/util.hh | 4 ++++ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e7c0700cf..990962763 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -7,11 +7,8 @@ #include #include -#include #include -#include #include -#include #include #include "util.hh" @@ -149,6 +146,15 @@ string baseNameOf(const Path & path) } +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + return st; +} + + bool pathExists(const Path & path) { int res; @@ -164,9 +170,7 @@ bool pathExists(const Path & path) Path readLink(const Path & path) { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); + struct stat st = lstat(path); if (!S_ISLNK(st.st_mode)) throw Error(format("`%1%' is not a symlink") % path); char buf[st.st_size]; @@ -178,9 +182,7 @@ Path readLink(const Path & path) bool isLink(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); + struct stat st = lstat(path); return S_ISLNK(st.st_mode); } @@ -269,9 +271,7 @@ static void _computePathSize(const Path & path, { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + struct stat st = lstat(path); bytes += st.st_size; blocks += st.st_blocks; @@ -301,9 +301,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed, printMsg(lvlVomit, format("%1%") % path); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + struct stat st = lstat(path); if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) { bytesFreed += st.st_size; @@ -350,9 +348,7 @@ void makePathReadOnly(const Path & path) { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + struct stat st = lstat(path); if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) { if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1) @@ -411,12 +407,18 @@ Paths createDirs(const Path & path) { Paths created; if (path == "/") return created; - if (!pathExists(path)) { + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1) + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) throw SysError(format("creating directory `%1%'") % path); + st = lstat(path); created.push_back(path); } + + if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path); + return created; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 27ad46904..f86290f31 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -4,6 +4,7 @@ #include "types.hh" #include +#include #include #include #include @@ -42,6 +43,9 @@ Path dirOf(const Path & path); following the final `/'. */ string baseNameOf(const Path & path); +/* Get status of `path'. */ +struct stat lstat(const Path & path); + /* Return true iff the given path exists. */ bool pathExists(const Path & path); From 5833243c92f28759ff0fc1ff9266535a3230e2d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 13:42:34 +0000 Subject: [PATCH 116/128] * Create /nix/var/nix/db if it's missing. --- src/libstore/local-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b80f66412..32c7a5330 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -199,6 +199,7 @@ LocalStore::LocalStore() Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); + createDirs(nixDBPath); Path gcRootsDir = nixStateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); From d787285af997a607bb678f39f340e663fafd3122 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 16:53:23 +0000 Subject: [PATCH 117/128] * nix-instantiate: return exit status 100 to denote a permanent build failure. The build hook can use this to distinguish between transient and permanent failures on the remote side. --- scripts/build-remote.pl.in | 14 +++++++------- src/libmain/shared.cc | 2 +- src/libstore/build.cc | 19 ++++++++++++++++--- src/libutil/types.hh | 5 +++-- src/libutil/util.cc | 3 ++- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index c551f6360..e943b0d9e 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -240,13 +240,13 @@ my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsD # work on some platforms when connection sharing is used.) pipe STDIN, DUMMY; # make sure we have a readable STDIN if (system("ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) { - # If we couldn't run ssh or there was an ssh problem (indicated by - # exit code 255), then we return exit code 1; otherwise we assume - # that the builder failed, which we indicate to Nix using exit - # code 100. It's important to distinguish between the two because - # the first is a transient failure and the latter is permanent. - my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100; - print STDERR "build of `$drvPath' on `$hostName' failed with exit code $?\n"; + # Note that if we get exit code 100 from `nix-store -r', it + # denotes a permanent build failure (as opposed to an SSH problem + # or a temporary Nix problem). We propagate this to the caller to + # allow it to distinguish between transient and permanent + # failures. + my $res = $? >> 8; + print STDERR "build of `$drvPath' on `$hostName' failed with exit code $res\n"; removeRoots; exit $res; } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7c2d92030..68f145820 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -393,7 +393,7 @@ int main(int argc, char * * argv) printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg()); if (e.prefix() != "" && !showTrace) printMsg(lvlError, "(use `--show-trace' to show detailed location information)"); - return 1; + return e.status; } catch (std::exception & e) { printMsg(lvlError, format("error: %1%") % e.what()); return 1; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0be9c42e0..83bd6754a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -214,6 +214,10 @@ public: bool cacheFailure; + /* Set if at least one derivation had a BuildError (i.e. permanent + failure). */ + bool permanentFailure; + LocalStore & store; boost::shared_ptr hook; @@ -266,7 +270,8 @@ public: /* Wait for input to become available. */ void waitForInput(); - + + unsigned int exitStatus(); }; @@ -1185,6 +1190,7 @@ void DerivationGoal::tryToBuild() if (printBuildTrace) printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % 0 % e.msg()); + worker.permanentFailure = true; amDone(ecFailed); return; } @@ -1321,6 +1327,7 @@ void DerivationGoal::buildDone() foreach (DerivationOutputs::iterator, i, drv.outputs) worker.store.registerFailedPath(i->second.path); + worker.permanentFailure = !hookError && !fixedOutput; amDone(ecFailed); return; } @@ -2444,6 +2451,7 @@ Worker::Worker(LocalStore & store) nrLocalBuilds = 0; lastWokenUp = 0; cacheFailure = queryBoolSetting("build-cache-failure", false); + permanentFailure = false; } @@ -2770,6 +2778,11 @@ void Worker::waitForInput() } +unsigned int Worker::exitStatus() +{ + return permanentFailure ? 100 : 1; +} + ////////////////////////////////////////////////////////////////////// @@ -2796,7 +2809,7 @@ void LocalStore::buildDerivations(const PathSet & drvPaths) } if (!failed.empty()) - throw Error(format("build of %1% failed") % showPaths(failed)); + throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus()); } @@ -2812,7 +2825,7 @@ void LocalStore::ensurePath(const Path & path) worker.run(goals); if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("path `%1%' does not exist and cannot be created") % path); + throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus()); } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 854a0f689..533fcca22 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -29,7 +29,8 @@ protected: string prefix_; // used for location traces etc. string err; public: - BaseError(const format & f); + unsigned int status; // exit status + BaseError(const format & f, unsigned int status = 1); ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } const string & msg() const throw () { return err; } @@ -41,7 +42,7 @@ public: class newClass : public superClass \ { \ public: \ - newClass(const format & f) : superClass(f) { }; \ + newClass(const format & f, unsigned int status = 1) : superClass(f, status) { }; \ }; MakeError(Error, BaseError) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 990962763..9adaac40d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -20,7 +20,8 @@ extern char * * environ; namespace nix { -BaseError::BaseError(const format & f) +BaseError::BaseError(const format & f, unsigned int status) + : status(status) { err = f.str(); } From 3dd02580e324d04ebfe999779978b4aa0f999ccd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Dec 2010 13:25:20 +0000 Subject: [PATCH 118/128] * I forgot to catch SQLiteBusy in registerValidPaths(). So registerValidPaths() now handles busy errors and registerValidPath() is simply a wrapper around it. --- src/libstore/local-store.cc | 67 +++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 32c7a5330..56d05c7bb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -463,7 +463,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) SQLiteStmtUse use(stmtRegisterValidPath); stmtRegisterValidPath.bind(info.path); stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime); + stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); if (info.deriver != "") stmtRegisterValidPath.bind(info.deriver); else @@ -506,31 +506,6 @@ void LocalStore::addReference(unsigned long long referrer, unsigned long long re } -void LocalStore::registerValidPath(const ValidPathInfo & info) -{ - assert(info.hash.type == htSHA256); - ValidPathInfo info2(info); - if (info2.registrationTime == 0) info2.registrationTime = time(0); - - while (1) { - try { - SQLiteTxn txn(db); - - unsigned long long id = addValidPath(info2); - - foreach (PathSet::const_iterator, i, info2.references) - addReference(id, queryValidPathId(*i)); - - txn.commit(); - break; - } catch (SQLiteBusy & e) { - /* Retry; the `txn' destructor will roll back the current - transaction. */ - } - } -} - - void LocalStore::registerFailedPath(const Path & path) { if (hasPathFailed(path)) return; @@ -896,22 +871,40 @@ Hash LocalStore::queryPathHash(const Path & path) } +void LocalStore::registerValidPath(const ValidPathInfo & info) +{ + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); +} + + void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - SQLiteTxn txn(db); + while (1) { + try { + SQLiteTxn txn(db); - foreach (ValidPathInfos::const_iterator, i, infos) - /* !!! Maybe the registration info should be updated if the - path is already valid. */ - if (!isValidPath(i->path)) addValidPath(*i); + foreach (ValidPathInfos::const_iterator, i, infos) { + assert(i->hash.type == htSHA256); + /* !!! Maybe the registration info should be updated if the + path is already valid. */ + if (!isValidPath(i->path)) addValidPath(*i); + } - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); + foreach (ValidPathInfos::const_iterator, i, infos) { + unsigned long long referrer = queryValidPathId(i->path); + foreach (PathSet::iterator, j, i->references) + addReference(referrer, queryValidPathId(*j)); + } + + txn.commit(); + break; + } catch (SQLiteBusy & e) { + /* Retry; the `txn' destructor will roll back the current + transaction. */ + } } - - txn.commit(); } From f1a6b97639474dbb13a1f96b4adf77c987a97d07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Dec 2010 08:39:37 +0000 Subject: [PATCH 119/128] * nix-copy-closure: make sure that the shell doesn't do globbing of `+' and `?' in filenames. This is very slow if /nix/store is very large. (This is a quick hack - a cleaner solution would be to bypass the shell entirely.) --- scripts/nix-copy-closure.in | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 1f1fded36..c037f003f 100644 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -61,7 +61,7 @@ if ($toMode) { # Copy TO the remote machine. my @allStorePaths; # Get the closure of this path. - my $pid = open(READ, "$binDir/nix-store --query --requisites @storePaths|") or die; + my $pid = open(READ, "set -f; $binDir/nix-store --query --requisites @storePaths|") or die; while () { chomp; @@ -73,7 +73,7 @@ if ($toMode) { # Copy TO the remote machine. # Ask the remote host which paths are invalid. - open(READ, "ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|"); + open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|"); my @missing = (); while () { chomp; @@ -88,7 +88,7 @@ if ($toMode) { # Copy TO the remote machine. print STDERR " $_\n" foreach @missing; my $extraOpts = ""; $extraOpts .= "--sign" if $sign == 1; - system("nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0 + system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0 or die "copying store paths to remote machine `$sshHost' failed: $?"; } @@ -101,7 +101,7 @@ else { # Copy FROM the remote machine. # machine. Paths are assumed to be store paths; there is no # resolution (following of symlinks). my $pid = open(READ, - "ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die; + "set -f; ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die; my @allStorePaths; @@ -115,7 +115,7 @@ else { # Copy FROM the remote machine. # What paths are already valid locally? - open(READ, "@bindir@/nix-store --check-validity --print-invalid @allStorePaths|"); + open(READ, "set -f; @bindir@/nix-store --check-validity --print-invalid @allStorePaths|"); my @missing = (); while () { chomp; @@ -130,7 +130,7 @@ else { # Copy FROM the remote machine. print STDERR " $_\n" foreach @missing; my $extraOpts = ""; $extraOpts .= "--sign" if $sign == 1; - system("ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0 + system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0 or die "copying store paths from remote machine `$sshHost' failed: $?"; } From a0be433fec792216ac5d9af68ec1fea6c21c7c1d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Dec 2010 14:25:54 +0000 Subject: [PATCH 120/128] * Disable X11 forwarding, it's not needed. --- scripts/SSH.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/SSH.pm b/scripts/SSH.pm index 3b546ca52..68f4a628b 100644 --- a/scripts/SSH.pm +++ b/scripts/SSH.pm @@ -3,6 +3,8 @@ use File::Temp qw(tempdir); our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or ""); +push @sshOpts, "-x"; + my $sshStarted = 0; my $sshHost; From eac93d6efeba68d9da95f01e10cd859eef6d85dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Dec 2010 09:28:51 +0000 Subject: [PATCH 121/128] * Use the right `make'. --- externals/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/Makefile.am b/externals/Makefile.am index 563f5424d..a5d67a91a 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -52,7 +52,7 @@ build-sqlite: $(SQLITE) touch build-sqlite install-exec-local:: build-sqlite - cd $(SQLITE) && make install + cd $(SQLITE) && $(MAKE) install rm -rf "$(DESTDIR)/$(pkglibdir)/dummy" endif From b1eb25217217087cb70a730da5311bd0890cf6ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Dec 2010 11:28:26 +0000 Subject: [PATCH 122/128] * Propagate the "100" exit status for failed builds through the Nix daemon. --- src/libstore/remote-store.cc | 7 +++++-- src/libstore/worker-protocol.hh | 2 +- src/nix-worker/nix-worker.cc | 5 +++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 26093a5d3..ae99846a3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -509,8 +509,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source) writeToStderr((const unsigned char *) s.c_str(), s.size()); } } - if (msg == STDERR_ERROR) - throw Error(readString(from)); + if (msg == STDERR_ERROR) { + string error = readString(from); + unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1; + throw Error(error, status); + } else if (msg != STDERR_LAST) throw Error("protocol error processing standard error"); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d77049bc7..acb8bc8b2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x107 +#define PROTOCOL_VERSION 0x108 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index ec5ca58aa..0fa1b40ae 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -178,7 +178,7 @@ static void startWork() /* stopWork() means that we're done; stop sending stderr to the client. */ -static void stopWork(bool success = true, const string & msg = "") +static void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) { /* Stop handling async client death; we're going to a state where we're either sending or receiving from the client, so we'll be @@ -192,6 +192,7 @@ static void stopWork(bool success = true, const string & msg = "") else { writeInt(STDERR_ERROR, to); writeString(msg, to); + if (status != 0) writeInt(status, to); } } @@ -637,7 +638,7 @@ static void processConnection() try { performOp(clientVersion, from, to, op); } catch (Error & e) { - stopWork(false, e.msg()); + stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); } assert(!canSendStderr); From c931a7aec5ccb2209d3c4bcf4452c807fe894d94 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Dec 2010 17:23:15 +0000 Subject: [PATCH 123/128] * Do a short sleep after SQLITE_BUSY. --- configure.ac | 3 +-- src/libstore/local-store.cc | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 2a8bedb0d..46e825899 100644 --- a/configure.ac +++ b/configure.ac @@ -286,8 +286,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) # Nice to have, but not essential. -AC_CHECK_FUNCS([strsignal]) -AC_CHECK_FUNCS([posix_fallocate]) +AC_CHECK_FUNCS([strsignal posix_fallocate nanosleep]) # This is needed if ATerm or bzip2 are static libraries, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 56d05c7bb..6af34cc77 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -35,6 +36,16 @@ static void throwSQLiteError(sqlite3 * db, const format & f) int err = sqlite3_errcode(db); if (err == SQLITE_BUSY) { printMsg(lvlError, "warning: SQLite database is busy"); + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = 100 * 1000 * 1000; /* 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); } else From d6c8b995c5c08a6c6a6b18f0b2cf5b4b95cfc1b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Dec 2010 10:32:32 +0000 Subject: [PATCH 124/128] * In Hydra manifests the Size field is missing, so don't rely on it. This caused a lot of "Use of uninitialized value" warnings from Perl. --- scripts/download-using-manifests.pl.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index 29321bff9..fe80bc11f 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -222,10 +222,10 @@ if ($ARGV[0] eq "--query") { my $u = $edge->{start}; my $v = $edge->{end}; if ($edge->{type} eq "patch") { - $downloadSize += $edge->{info}->{size}; + $downloadSize += $edge->{info}->{size} || 0; } elsif ($edge->{type} eq "narfile") { - $downloadSize += $edge->{info}->{size}; + $downloadSize += $edge->{info}->{size} || 0; } } From 9db190eb31d4adc412d50bc03574951f9a1f9dae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 14 Jan 2011 12:47:10 +0000 Subject: [PATCH 125/128] * builtins.substring: if "start" is beyond the end of the string, return the empty string. --- src/libexpr/primops.cc | 2 +- tests/lang/eval-okay-substring.exp | 2 +- tests/lang/eval-okay-substring.nix | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a4812de06..3e955ea3f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -965,7 +965,7 @@ static void prim_substring(EvalState & state, Value * * args, Value & v) if (start < 0) throw EvalError("negative start position in `substring'"); - mkString(v, string(s, start, len), context); + mkString(v, start >= s.size() ? "" : string(s, start, len), context); } diff --git a/tests/lang/eval-okay-substring.exp b/tests/lang/eval-okay-substring.exp index d936b7e96..6aace04b0 100644 --- a/tests/lang/eval-okay-substring.exp +++ b/tests/lang/eval-okay-substring.exp @@ -1 +1 @@ -"ooxfoobarybarzobaabb" +"ooxfoobarybarzobaabbc" diff --git a/tests/lang/eval-okay-substring.nix b/tests/lang/eval-okay-substring.nix index 184d72580..424af00d9 100644 --- a/tests/lang/eval-okay-substring.nix +++ b/tests/lang/eval-okay-substring.nix @@ -17,3 +17,5 @@ substring 1 2 s + substring 3 0 s + "b" + substring 3 1 s ++ "c" ++ substring 5 10 "perl" From aeae0beba49e98655861c5c50e573bd7d9cfb5c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 5 Feb 2011 16:29:10 +0000 Subject: [PATCH 126/128] * Clang compatibility fix. Clang actually checks template definitions when they are defined --- src/libmain/shared.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index f70f6893b..c99810c78 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,7 +1,7 @@ #ifndef __SHARED_H #define __SHARED_H -#include "types.hh" +#include "util.hh" #include From 0304fda3cfdbd1e0cbc939b667cbe833eb726be1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 5 Feb 2011 16:40:15 +0000 Subject: [PATCH 127/128] * Propagate the CC setting. --- externals/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/Makefile.am b/externals/Makefile.am index a5d67a91a..884d87bf1 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -17,7 +17,7 @@ build-bzip2: else build-bzip2: $(BZIP2) (cd $(BZIP2) && \ - $(MAKE) && \ + $(MAKE) CC="$(CC)" && \ $(MAKE) install PREFIX=$(abs_builddir)/inst-bzip2) touch build-bzip2 From c0340eec5ac175f7b757d883aca71d1a4382c641 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Feb 2011 12:30:28 +0000 Subject: [PATCH 128/128] --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 46e825899..e34e4ba3a 100644 --- a/configure.ac +++ b/configure.ac @@ -247,7 +247,7 @@ AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH], [prefix of SQLite]), sqlite=$withval, sqlite=) AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite") -SQLITE_VERSION=3070400 +SQLITE_VERSION=3070500 AC_SUBST(SQLITE_VERSION) if test -z "$sqlite"; then sqlite_lib='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)/libsqlite3.la'