diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index a1f4a4b75..291b56448 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-2.3"> -Release 2.3 (2019-08-??) +Release 2.3 (2019-09-04) This is primarily a bug fix release. However, it makes some incompatible changes: diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 92ed12fa5..b340610e9 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -13,8 +13,12 @@ RunAtLoad - Program - @bindir@/nix-daemon + ProgramArguments + + /bin/sh + -c + /bin/wait4path @bindir@/nix-daemon && @bindir@/nix-daemon + StandardErrorPath /var/log/nix-daemon.log StandardOutPath diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..10cde8704 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -55,7 +55,7 @@ void BinaryCacheStore::init() } void BinaryCacheStore::getFile(const std::string & path, - Callback> callback) + Callback> callback) noexcept { try { callback(getFile(path)); @@ -240,7 +240,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback> callback) + Callback> callback) noexcept { auto uri = getUri(); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, @@ -249,21 +249,23 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, auto narInfoFile = narInfoFileFor(storePath); + auto callbackPtr = std::make_shared(std::move(callback)); + getFile(narInfoFile, {[=](std::future> fut) { try { auto data = fut.get(); - if (!data) return callback(nullptr); + if (!data) return (*callbackPtr)(nullptr); stats.narInfoRead++; - callback((std::shared_ptr) + (*callbackPtr)((std::shared_ptr) std::make_shared(*this, *data, narInfoFile)); (void) act; // force Activity into this lambda to ensure it stays alive } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 953f3b90a..af108880c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -47,7 +47,7 @@ public: /* Fetch the specified file and call the specified callback with the result. A subclass may implement this asynchronously. */ virtual void getFile(const std::string & path, - Callback> callback); + Callback> callback) noexcept; std::shared_ptr getFile(const std::string & path); @@ -73,7 +73,7 @@ public: bool isValidPathUncached(const Path & path) override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; Path queryPathFromHashPart(const string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 19cb9addc..38762ad0d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1197,7 +1197,7 @@ void DerivationGoal::haveDerivation() /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && drv->substitutesAllowed()) + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3961126ff..23fcfb281 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const } -bool BasicDerivation::substitutesAllowed() const -{ - return get(env, "allowSubstitutes", "1") == "1"; -} - - bool BasicDerivation::isBuiltin() const { return string(builder, 0, 8) == "builtin:"; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9753e796d..8e02c9bc5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -56,8 +56,6 @@ struct BasicDerivation the given derivation. */ Path findOutput(const string & id) const; - bool substitutesAllowed() const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 3835727a0..a5165773a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -77,13 +77,13 @@ struct CurlDownloader : public Downloader DownloadItem(CurlDownloader & downloader, const DownloadRequest & request, - Callback callback) + Callback && callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) - , callback(callback) + , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { writtenToSink += len; @@ -342,15 +342,9 @@ struct CurlDownloader : public Downloader (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); done = true; - - try { - act.progress(result.bodySize, result.bodySize); - callback(std::move(result)); - } catch (...) { - done = true; - callback.rethrow(); - } + callback(std::move(result)); } else { @@ -671,7 +665,7 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared(*this, request, callback)); + enqueueItem(std::make_shared(*this, request, std::move(callback))); } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ea39d4f22..c8a00a949 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -131,23 +131,25 @@ protected: } void getFile(const std::string & path, - Callback> callback) override + Callback> callback) noexcept override { checkEnabled(); auto request(makeRequest(path)); + auto callbackPtr = std::make_shared(std::move(callback)); + getDownloader()->enqueueDownload(request, - {[callback, this](std::future result) { + {[callbackPtr, this](std::future result) { try { - callback(result.get().data); + (*callbackPtr)(result.get().data); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); + return (*callbackPtr)(std::shared_ptr()); maybeDisable(); - callback.rethrow(); + callbackPtr->rethrow(); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7c9bc2b68..d5fbdd25a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,7 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - Callback> callback) override + Callback> callback) noexcept override { try { auto conn(connections->get()); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a9399130c..1062df781 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -622,7 +622,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - Callback> callback) + Callback> callback) noexcept { try { auto info = std::make_shared(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index af8b84bf5..3ae34c403 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index adcce026f..dddf13430 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,5 @@ #include "derivations.hh" +#include "parsed-derivations.hh" #include "globals.hh" #include "local-store.hh" #include "store-api.hh" @@ -189,6 +190,7 @@ void Store::queryMissing(const PathSet & targets, } Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); PathSet invalid; for (auto & j : drv.outputs) @@ -197,7 +199,7 @@ void Store::queryMissing(const PathSet & targets, invalid.insert(j.second.path); if (invalid.empty()) return; - if (settings.useSubstitutes && drv.substitutesAllowed()) { + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, i2.first, make_ref(drv), output, drvState)); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 17fde00a0..87be8a24e 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -108,4 +108,9 @@ bool ParsedDerivation::willBuildLocally() const return getBoolAttr("preferLocalBuild") && canBuildLocally(); } +bool ParsedDerivation::substitutesAllowed() const +{ + return getBoolAttr("allowSubstitutes", true); +} + } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index ed07dc652..9bde4b4dc 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -30,6 +30,8 @@ public: bool canBuildLocally() const; bool willBuildLocally() const; + + bool substitutesAllowed() const; }; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1c2e23f9c..e38fe49a7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -349,7 +349,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - Callback> callback) + Callback> callback) noexcept { try { std::shared_ptr info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 80f18ab71..82fbec092 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -41,7 +41,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d007af037..e8c1c1fdd 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -329,13 +329,14 @@ ref Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, - Callback> callback) + Callback> callback) noexcept { - assertStorePath(storePath); - - auto hashPart = storePathToHash(storePath); + std::string hashPart; try { + assertStorePath(storePath); + + hashPart = storePathToHash(storePath); { auto res = state.lock()->pathInfoCache.get(hashPart); @@ -365,8 +366,10 @@ void Store::queryPathInfo(const Path & storePath, } catch (...) { return callback.rethrow(); } + auto callbackPtr = std::make_shared(std::move(callback)); + queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callback](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -386,8 +389,8 @@ void Store::queryPathInfo(const Path & storePath, throw InvalidPath("path '%s' is not valid", storePath); } - callback(ref(info)); - } catch (...) { callback.rethrow(); } + (*callbackPtr)(ref(info)); + } catch (...) { callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e5e5c182c..5733bd9de 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -361,12 +361,12 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - Callback> callback); + Callback> callback) noexcept; protected: virtual void queryPathInfoUncached(const Path & path, - Callback> callback) = 0; + Callback> callback) noexcept = 0; public: diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1f3d1e2ae..2bf9725af 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -469,21 +469,34 @@ string get(const T & map, const string & key, const string & def = "") type T or an exception. (We abuse std::future to pass the value or exception.) */ template -struct Callback +class Callback { std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: Callback(std::function)> fun) : fun(fun) { } - void operator()(T && t) const + Callback(Callback && callback) : fun(std::move(callback.fun)) { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); std::promise promise; promise.set_value(std::move(t)); fun(promise.get_future()); } - void rethrow(const std::exception_ptr & exc = std::current_exception()) const + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept { + auto prev = done.test_and_set(); + assert(!prev); std::promise promise; promise.set_exception(exc); fun(promise.get_future()); diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 06f5f0795..a74c675b0 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -17,42 +17,50 @@ expr=$(cat < \$out/bar echo 1... sleep 2 - echo 200 > $fake_free + echo 200 > ${fake_free}.tmp1 + mv ${fake_free}.tmp1 $fake_free echo 2... sleep 2 echo 3... + sleep 2 + echo 4... [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] ''; } EOF ) +expr2=$(cat < \$out/bar + echo 1... + sleep 2 + echo 200 > ${fake_free}.tmp2 + mv ${fake_free}.tmp2 $fake_free + echo 2... + sleep 2 + echo 3... + sleep 2 + echo 4... + ''; +} +EOF +) + nix build --impure -v -o $TEST_ROOT/result-A -L "($expr)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 & pid=$! -expr2=$(cat < \$out/bar - echo 1... - sleep 2 - echo 200 > $fake_free - echo 2... - sleep 2 - echo 3... - ''; -} -EOF -) - nix build --impure -v -o $TEST_ROOT/result-B -L "($expr2)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1