From a3e6415ba8cf1b8d2a1db40c06997d997eac8afc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Dec 2006 23:05:01 +0000 Subject: [PATCH] * New primop builtins.filterSource, which can be used to filter files from a source directory. All files for which a predicate function returns true are copied to the store. Typical example is to leave out the .svn directory: stdenv.mkDerivation { ... src = builtins.filterSource (path: baseNameOf (toString path) != ".svn") ./source-dir; # as opposed to # src = ./source-dir; } This is important because the .svn directory influences the hash in a rather unpredictable and variable way. --- doc/manual/release-notes.xml | 7 +++++- src/libexpr/primops.cc | 38 ++++++++++++++++++++++++++++++++ src/libstore/build.cc | 2 +- src/libstore/local-store.cc | 42 +++++++----------------------------- src/libstore/local-store.hh | 3 ++- src/libstore/remote-store.cc | 4 ++-- src/libstore/remote-store.hh | 3 ++- src/libstore/store-api.cc | 6 +++--- src/libstore/store-api.hh | 10 ++++++--- src/libutil/archive.cc | 21 +++++++++--------- src/libutil/archive.hh | 8 +++---- src/libutil/hash.cc | 4 ++-- src/libutil/hash.hh | 5 ++++- src/libutil/serialise.hh | 27 +++++++++++++++++++++++ src/libutil/util.hh | 4 ++-- src/nix-worker/nix-worker.cc | 3 +-- tests/Makefile.am | 4 +++- tests/filter-source.nix.in | 7 ++++++ tests/filter-source.sh | 13 +++++++++++ 19 files changed, 143 insertions(+), 68 deletions(-) create mode 100644 tests/filter-source.nix.in create mode 100644 tests/filter-source.sh diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index 60f645485..ba36d949d 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -12,7 +12,7 @@ - TODO: multi-user support. + TODO: multi-user support. nix-store has a new operation @@ -50,6 +50,11 @@ . + TODO: new built-ins + builtins.attrNames, + builtins.filterSource. + + diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fcf035450..84d9f5a13 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "store-api.hh" #include "util.hh" +#include "archive.hh" #include "expr-to-xml.hh" #include "nixexpr-ast.hh" @@ -726,6 +727,42 @@ static Expr primLessThan(EvalState & state, const ATermVector & args) } +struct FilterFromExpr : PathFilter +{ + EvalState & state; + Expr filter; + + FilterFromExpr(EvalState & state, Expr filter) + : state(state), filter(filter) + { + } + + bool operator () (const Path & path) + { + printMsg(lvlError, format("filter %1%") % path); + Expr call = makeCall(filter, makePath(toATerm(path))); + return evalBool(state, call); + } +}; + + +static Expr primFilterSource(EvalState & state, const ATermVector & args) +{ + PathSet context; + Path path = coerceToPath(state, args[1], context); + if (!context.empty()) + throw EvalError(format("string `%1%' cannot refer to other paths") % path); + + FilterFromExpr filter(state, args[0]); + + Path dstPath = readOnlyMode + ? computeStorePathForPath(path, false, false, "", filter).first + : store->addToStore(path, false, false, "", filter); + + return makeStr(dstPath, singleton(dstPath)); +} + + void EvalState::addPrimOps() { addPrimOp("builtins", 0, primBuiltins); @@ -762,6 +799,7 @@ void EvalState::addPrimOps() addPrimOp("__add", 2, primAdd); addPrimOp("__lessThan", 2, primLessThan); addPrimOp("__toFile", 2, primToFile); + addPrimOp("__filterSource", 2, primFilterSource); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 14388a7b5..9bcf336a4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -949,7 +949,7 @@ void DerivationGoal::buildDone() as that means that someone else can have interfered with the build. Also, the output should be owned by the build user. */ - if ((st.st_mode & (S_IWGRP | S_IWOTH)) || + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser.enabled() && st.st_uid != buildUser.getUID())) throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path); #endif diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 01d1e398c..1bed672d2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -171,33 +171,7 @@ void createStoreTransaction(Transaction & txn) } -/* Path copying. */ - -struct CopySink : Sink -{ - string s; - virtual void operator () (const unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } -}; - - -struct CopySource : Source -{ - string & s; - unsigned int pos; - CopySource(string & _s) : s(_s), pos(0) { } - virtual void operator () (unsigned char * data, unsigned int len) - { - s.copy((char *) data, len, pos); - pos += len; - assert(pos <= s.size()); - } -}; - - -void copyPath(const Path & src, const Path & dst) +void copyPath(const Path & src, const Path & dst, PathFilter & filter) { debug(format("copying `%1%' to `%2%'") % src % dst); @@ -206,10 +180,10 @@ void copyPath(const Path & src, const Path & dst) for very large paths, but `copyPath' is mainly used for small files. */ - CopySink sink; - dumpPath(src, sink); + StringSink sink; + dumpPath(src, sink, filter); - CopySource source(sink.s); + StringSource source(sink.s); restorePath(dst, source); } @@ -646,13 +620,13 @@ static void invalidatePath(Transaction & txn, const Path & path) Path LocalStore::addToStore(const Path & _srcPath, bool fixed, - bool recursive, string hashAlgo) + bool recursive, string hashAlgo, PathFilter & filter) { Path srcPath(absPath(_srcPath)); debug(format("adding `%1%' to the store") % srcPath); std::pair pr = - computeStorePathForPath(srcPath, fixed, recursive, hashAlgo); + computeStorePathForPath(srcPath, fixed, recursive, hashAlgo, filter); Path & dstPath(pr.first); Hash & h(pr.second); @@ -669,9 +643,9 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed, if (pathExists(dstPath)) deletePathWrapped(dstPath); - copyPath(srcPath, dstPath); + copyPath(srcPath, dstPath, filter); - Hash h2 = hashPath(htSHA256, dstPath); + Hash h2 = hashPath(htSHA256, dstPath, filter); if (h != h2) throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)") % srcPath % dstPath % printHash(h) % printHash(h2)); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ef508630b..8f4ed8fc8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -50,7 +50,8 @@ public: void queryReferrers(const Path & path, PathSet & referrers); Path addToStore(const Path & srcPath, bool fixed = false, - bool recursive = false, string hashAlgo = ""); + bool recursive = false, string hashAlgo = "", + PathFilter & filter = defaultPathFilter); Path addTextToStore(const string & suffix, const string & s, const PathSet & references); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 504eb5232..e6b34c9b8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -213,7 +213,7 @@ void RemoteStore::queryReferrers(const Path & path, Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, - bool recursive, string hashAlgo) + bool recursive, string hashAlgo, PathFilter & filter) { Path srcPath(absPath(_srcPath)); @@ -222,7 +222,7 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, writeInt(fixed ? 1 : 0, to); writeInt(recursive ? 1 : 0, to); writeString(hashAlgo, to); - dumpPath(srcPath, to); + dumpPath(srcPath, to, filter); processStderr(); Path path = readStorePath(from); return path; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 82cfb08e9..42d4e94e2 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -38,7 +38,8 @@ public: void queryReferrers(const Path & path, PathSet & referrers); Path addToStore(const Path & srcPath, bool fixed = false, - bool recursive = false, string hashAlgo = ""); + bool recursive = false, string hashAlgo = "", + PathFilter & filter = defaultPathFilter); Path addTextToStore(const string & suffix, const string & s, const PathSet & references); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 677c3ca3e..be9ea788b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -94,9 +94,9 @@ Path makeFixedOutputPath(bool recursive, std::pair computeStorePathForPath(const Path & srcPath, - bool fixed, bool recursive, string hashAlgo) + bool fixed, bool recursive, string hashAlgo, PathFilter & filter) { - Hash h = hashPath(htSHA256, srcPath); + Hash h = hashPath(htSHA256, srcPath, filter); string baseName = baseNameOf(srcPath); @@ -104,7 +104,7 @@ std::pair computeStorePathForPath(const Path & srcPath, if (fixed) { HashType ht(parseHashType(hashAlgo)); - Hash h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath); + Hash h2 = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath); dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 19c5d81cf..d92b03df0 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -79,9 +79,12 @@ public: /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. If `fixed' is true, then the output of a fixed-output - derivation is pre-loaded into the Nix store. */ + derivation is pre-loaded into the Nix store. The function + object `filter' can be used to exclude files (see + libutil/archive.hh). */ virtual Path addToStore(const Path & srcPath, bool fixed = false, - bool recursive = false, string hashAlgo = "") = 0; + bool recursive = false, string hashAlgo = "", + PathFilter & filter = defaultPathFilter) = 0; /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ @@ -195,7 +198,8 @@ Path makeFixedOutputPath(bool recursive, Returns the store path and the cryptographic hash of the contents of srcPath. */ std::pair computeStorePathForPath(const Path & srcPath, - bool fixed = false, bool recursive = false, string hashAlgo = ""); + bool fixed = false, bool recursive = false, string hashAlgo = "", + PathFilter & filter = defaultPathFilter); /* Preparatory part of addTextToStore(). diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 4aedd31f7..a8e018b69 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -18,28 +18,29 @@ namespace nix { static string archiveVersion1 = "nix-archive-1"; -DumpFilter defaultDumpFilter; +PathFilter defaultPathFilter; -static void dump(const string & path, Sink & sink, DumpFilter & filter); +static void dump(const string & path, Sink & sink, PathFilter & filter); -static void dumpEntries(const Path & path, Sink & sink, DumpFilter & filter) +static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) { Strings names = readDirectory(path); vector names2(names.begin(), names.end()); sort(names2.begin(), names2.end()); - for (vector::iterator it = names2.begin(); - it != names2.end(); it++) + for (vector::iterator i = names2.begin(); + i != names2.end(); ++i) { - if (filter(path)) { + Path entry = path + "/" + *i; + if (filter(entry)) { writeString("entry", sink); writeString("(", sink); writeString("name", sink); - writeString(*it, sink); + writeString(*i, sink); writeString("node", sink); - dump(path + "/" + *it, sink, filter); + dump(entry, sink, filter); writeString(")", sink); } } @@ -69,7 +70,7 @@ static void dumpContents(const Path & path, unsigned int size, } -static void dump(const Path & path, Sink & sink, DumpFilter & filter) +static void dump(const Path & path, Sink & sink, PathFilter & filter) { struct stat st; if (lstat(path.c_str(), &st)) @@ -106,7 +107,7 @@ static void dump(const Path & path, Sink & sink, DumpFilter & filter) } -void dumpPath(const Path & path, Sink & sink, DumpFilter & filter) +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) { writeString(archiveVersion1, sink); dump(path, sink, filter); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 70e836055..5f85e1beb 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -45,16 +45,16 @@ namespace nix { `+' denotes string concatenation. */ -struct DumpFilter +struct PathFilter { - virtual ~DumpFilter() { } + virtual ~PathFilter() { } virtual bool operator () (const Path & path) { return true; } }; -extern DumpFilter defaultDumpFilter; +extern PathFilter defaultPathFilter; void dumpPath(const Path & path, Sink & sink, - DumpFilter & filter = defaultDumpFilter); + PathFilter & filter = defaultPathFilter); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 8dc33f5d0..262dbef20 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -294,13 +294,13 @@ struct HashSink : Sink }; -Hash hashPath(HashType ht, const Path & path) +Hash hashPath(HashType ht, const Path & path, PathFilter & filter) { HashSink sink; sink.ht = ht; Hash hash(ht); start(ht, sink.ctx); - dumpPath(path, sink); + dumpPath(path, sink, filter); finish(ht, sink.ctx, hash.hash); return hash; } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 74ae51db3..78227fb6a 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -69,7 +69,10 @@ Hash hashFile(HashType ht, const Path & path); /* Compute the hash of the given path. The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ -Hash hashPath(HashType ht, const Path & path); +struct PathFilter; +extern PathFilter defaultPathFilter; +Hash hashPath(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); /* Compress a hash to the specified number of bytes by cyclically XORing bytes together. */ diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index fe3492235..c18e82463 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -66,6 +66,33 @@ struct FdSource : Source }; +/* A sink that writes data to a string. */ +struct StringSink : Sink +{ + string s; + virtual void operator () (const unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } +}; + + +/* A source that reads data from a string. */ +struct StringSource : Source +{ + string & s; + unsigned int pos; + StringSource(string & _s) : s(_s), pos(0) { } + virtual void operator () (unsigned char * data, unsigned int len) + { + s.copy((char *) data, len, pos); + pos += len; + if (pos > s.size()) + throw Error("end of string reached"); + } +}; + + void writePadding(unsigned int len, Sink & sink); void writeInt(unsigned int n, Sink & sink); void writeString(const string & s, Sink & sink); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8f79ec9be..1cc97145c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -164,10 +164,10 @@ struct AutoDeleteArray class AutoDelete { - string path; + Path path; bool del; public: - AutoDelete(const string & p); + AutoDelete(const Path & p); ~AutoDelete(); void cancel(); }; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 0be4b8e64..17fbbf264 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -240,6 +240,7 @@ static void performOp(Source & from, Sink & to, unsigned int op) string hashAlgo = readString(from); Path tmp = createTempDir(); + AutoDelete delTmp(tmp); Path tmp2 = tmp + "/" + baseName; restorePath(tmp2, from); @@ -248,8 +249,6 @@ static void performOp(Source & from, Sink & to, unsigned int op) stopWork(); writeString(path, to); - - deletePath(tmp); break; } diff --git a/tests/Makefile.am b/tests/Makefile.am index 4b56d0226..d4c39ca5f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -15,12 +15,13 @@ user-envs.sh: user-envs.nix fixed.sh: fixed.nix gc-runtime.sh: gc-runtime.nix check-refs.sh: check-refs.nix +filter-source.sh: filter-source.nix TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ locking.sh parallel.sh build-hook.sh substitutes.sh substitutes2.sh \ fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ - gc-runtime.sh install-package.sh check-refs.sh + gc-runtime.sh install-package.sh check-refs.sh filter-source.sh XFAIL_TESTS = @@ -44,5 +45,6 @@ EXTRA_DIST = $(TESTS) \ fixed.nix.in fixed.builder1.sh fixed.builder2.sh \ gc-runtime.nix.in gc-runtime.builder.sh \ check-refs.nix.in \ + filter-source.nix.in \ $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) \ common.sh.in diff --git a/tests/filter-source.nix.in b/tests/filter-source.nix.in new file mode 100644 index 000000000..2b5c23d31 --- /dev/null +++ b/tests/filter-source.nix.in @@ -0,0 +1,7 @@ +derivation { + name = "filter"; + system = "@system@"; + builder = "@shell@"; + args = ["-e" "-x" (builtins.toFile "builder" "PATH=@testPath@; ln -s $input $out")]; + input = builtins.filterSource (path: baseNameOf (toString path) != "foo") ./test-tmp/filterin; +} diff --git a/tests/filter-source.sh b/tests/filter-source.sh new file mode 100644 index 000000000..4880969ba --- /dev/null +++ b/tests/filter-source.sh @@ -0,0 +1,13 @@ +source common.sh + +rm -rf $TEST_ROOT/filterin +mkdir $TEST_ROOT/filterin +mkdir $TEST_ROOT/filterin/foo +touch $TEST_ROOT/filterin/foo/bar +touch $TEST_ROOT/filterin/xyzzy + +$NIX_BIN_DIR/nix-build ./filter-source.nix -o $TEST_ROOT/filterout + +set -x +test ! -e $TEST_ROOT/filterout/foo/bar +test -e $TEST_ROOT/filterout/xyzzy