From 95f47c28fb8786f8d8d529192465bb6ec20db46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 3 Jun 2022 17:01:16 +0200 Subject: [PATCH 01/58] Make nix copy parallel again FILLME --- src/libstore/store-api.cc | 203 ++++++++++++++++++++------------------ src/libstore/store-api.hh | 7 ++ 2 files changed, 115 insertions(+), 95 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8861274a2..008451666 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -258,6 +258,86 @@ StorePath Store::addToStore( return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } +void Store::addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + std::atomic nrDone{0}; + std::atomic nrFailed{0}; + std::atomic bytesExpected{0}; + std::atomic nrRunning{0}; + + using PathWithInfo = std::pair>; + + std::map infosMap; + StorePathSet storePathsToAdd; + for (auto & thingToAdd : pathsToCopy) { + infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); + storePathsToAdd.insert(thingToAdd.first.path); + } + + auto showProgress = [&]() { + act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed); + }; + + ThreadPool pool; + + processGraph(pool, + storePathsToAdd, + + [&](const StorePath & path) { + auto & [info, source] = *infosMap.at(path); + /* auto storePathForDst = info.storePath; */ + /* if (info->ca && info->references.empty()) { */ + /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ + /* if (dstStore.storeDir == srcStore.storeDir) */ + /* assert(storePathForDst == storePath); */ + /* if (storePathForDst != storePath) */ + /* debug("replaced path '%s' to '%s' for substituter '%s'", */ + /* srcStore.printStorePath(storePath), */ + /* dstStore.printStorePath(storePathForDst), */ + /* dstStore.getUri()); */ + /* } */ + /* pathsMap.insert_or_assign(storePath, storePathForDst); */ + + if (isValidPath(info.path)) { + nrDone++; + showProgress(); + return StorePathSet(); + } + + bytesExpected += info.narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info.references; + }, + + [&](const StorePath & path) { + checkInterrupt(); + + auto & [info, source] = *infosMap.at(path); + + if (!isValidPath(info.path)) { + MaintainCount mc(nrRunning); + showProgress(); + try { + addToStore(info, *source, repair, checkSigs); + } catch (Error &e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what()); + showProgress(); + return; + } + } + + nrDone++; + showProgress(); + }); +} void Store::addMultipleToStore( Source & source, @@ -998,106 +1078,39 @@ std::map copyPaths( Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); - auto sorted = srcStore.topoSortPaths(missing); - std::reverse(sorted.begin(), sorted.end()); + /* auto sorted = srcStore.topoSortPaths(missing); */ + /* std::reverse(sorted.begin(), sorted.end()); */ - auto source = sinkToSource([&](Sink & sink) { - sink << sorted.size(); - for (auto & storePath : sorted) { - auto srcUri = srcStore.getUri(); - auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(storePath); - Activity act(*logger, lvlInfo, actCopyPath, - makeCopyPathMessage(srcUri, dstUri, storePathS), - {storePathS, srcUri, dstUri}); - PushActivity pact(act.id); + /* auto source = sinkToSource([&](Sink & sink) { */ + /* sink << sorted.size(); */ + /* for (auto & storePath : sorted) { */ + /* auto srcUri = srcStore.getUri(); */ + /* auto dstUri = dstStore.getUri(); */ + /* auto storePathS = srcStore.printStorePath(storePath); */ + /* Activity act(*logger, lvlInfo, actCopyPath, */ + /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ + /* {storePathS, srcUri, dstUri}); */ + /* PushActivity pact(act.id); */ - auto info = srcStore.queryPathInfo(storePath); - info->write(sink, srcStore, 16); - srcStore.narFromPath(storePath, sink); - } - }); + /* auto info = srcStore.queryPathInfo(storePath); */ + /* info->write(sink, srcStore, 16); */ + /* srcStore.narFromPath(storePath, sink); */ + /* } */ + /* }); */ - dstStore.addMultipleToStore(*source, repair, checkSigs); + std::vector>> pathsToCopy; + + for (auto & missingPath : missing) { + auto info = srcStore.queryPathInfo(missingPath); + auto source = sinkToSource([&](Sink & sink) { + srcStore.narFromPath(missingPath, sink); + }); + pathsToCopy.push_back(std::pair{*info, std::move(source)}); + } + + dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); #if 0 - std::atomic nrDone{0}; - std::atomic nrFailed{0}; - std::atomic bytesExpected{0}; - std::atomic nrRunning{0}; - - auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning, nrFailed); - }; - - ThreadPool pool; - - processGraph(pool, - StorePathSet(missing.begin(), missing.end()), - - [&](const StorePath & storePath) { - auto info = srcStore.queryPathInfo(storePath); - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (dstStore.isValidPath(storePath)) { - nrDone++; - showProgress(); - return StorePathSet(); - } - - bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); - - return info->references; - }, - - [&](const StorePath & storePath) { - checkInterrupt(); - - auto info = srcStore.queryPathInfo(storePath); - - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (!dstStore.isValidPath(storePathForDst)) { - MaintainCount mc(nrRunning); - showProgress(); - try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } catch (Error &e) { - nrFailed++; - if (!settings.keepGoing) - throw e; - printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); - showProgress(); - return; - } - } - - nrDone++; - showProgress(); - }); #endif return pathsMap; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0c8a4db56..d934979cf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "nar-info.hh" #include "realisation.hh" #include "path.hh" #include "derived-path.hh" @@ -364,6 +365,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs + ); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. From cb0553ecd0122f693653c1ac82beb26d127ce3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 14:03:46 +0200 Subject: [PATCH 02/58] Restore the "low-latency" ssh copying --- src/libstore/remote-store.cc | 17 +++++++++++++ src/libstore/remote-store.hh | 8 ++++++ src/libstore/store-api.cc | 49 ++++++++++++++++-------------------- src/libstore/store-api.hh | 6 ++++- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..ad2e5c18a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -673,6 +673,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } +void RemoteStore::addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto source = sinkToSource([&](Sink & sink) { + sink << pathsToCopy.size(); + for (auto & [pathInfo, pathSource] : pathsToCopy) { + pathInfo.write(sink, *this, 16); + pathSource->drainInto(sink); + } + }); + + addMultipleToStore(*source, repair, checkSigs); +} + void RemoteStore::addMultipleToStore( Source & source, RepairFlag repair, diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8493be6fc..5a599997e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -88,6 +88,14 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; + void addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; + + + StorePath addTextToStore( std::string_view name, std::string_view s, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 008451666..eeec6c6f7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -259,7 +259,7 @@ StorePath Store::addToStore( } void Store::addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) @@ -1072,37 +1072,33 @@ std::map copyPaths( for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + + // In the general case, `addMultipleToStore` requires a sorted list of + // store paths to add, so sort them right now + auto sortedMissing = srcStore.topoSortPaths(missing); + std::reverse(sortedMissing.begin(), sortedMissing.end()); + std::map pathsMap; for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + Store::PathsSource pathsToCopy; - /* auto sorted = srcStore.topoSortPaths(missing); */ - /* std::reverse(sorted.begin(), sorted.end()); */ - - /* auto source = sinkToSource([&](Sink & sink) { */ - /* sink << sorted.size(); */ - /* for (auto & storePath : sorted) { */ - /* auto srcUri = srcStore.getUri(); */ - /* auto dstUri = dstStore.getUri(); */ - /* auto storePathS = srcStore.printStorePath(storePath); */ - /* Activity act(*logger, lvlInfo, actCopyPath, */ - /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ - /* {storePathS, srcUri, dstUri}); */ - /* PushActivity pact(act.id); */ - - /* auto info = srcStore.queryPathInfo(storePath); */ - /* info->write(sink, srcStore, 16); */ - /* srcStore.narFromPath(storePath, sink); */ - /* } */ - /* }); */ - - std::vector>> pathsToCopy; - - for (auto & missingPath : missing) { + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); auto source = sinkToSource([&](Sink & sink) { + + // We can reasonably assume that the copy will happen whenever we + // read the path, so log something about that at that point + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(missingPath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + srcStore.narFromPath(missingPath, sink); }); pathsToCopy.push_back(std::pair{*info, std::move(source)}); @@ -1110,9 +1106,6 @@ std::map copyPaths( dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); - #if 0 - #endif - return pathsMap; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d934979cf..c0a61115b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,13 +360,17 @@ public: virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; + // A list of paths infos along with a source providing the content of the + // associated store path + using PathsSource = std::vector>>; + /* Import multiple paths into the store. */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); virtual void addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs From 480c2b6699e6afc6a746ba8a72319f197c3556ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 15:13:11 +0200 Subject: [PATCH 03/58] Rewrite the CA paths when moving them between store Bring back the possibility to copy CA paths with no reference (like the outputs of FO derivations or stuff imported at eval time) between stores that have a different prefix. --- src/libstore/store-api.cc | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index eeec6c6f7..61a12e84a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -289,18 +289,6 @@ void Store::addMultipleToStore( [&](const StorePath & path) { auto & [info, source] = *infosMap.at(path); - /* auto storePathForDst = info.storePath; */ - /* if (info->ca && info->references.empty()) { */ - /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ - /* if (dstStore.storeDir == srcStore.storeDir) */ - /* assert(storePathForDst == storePath); */ - /* if (storePathForDst != storePath) */ - /* debug("replaced path '%s' to '%s' for substituter '%s'", */ - /* srcStore.printStorePath(storePath), */ - /* dstStore.printStorePath(storePathForDst), */ - /* dstStore.getUri()); */ - /* } */ - /* pathsMap.insert_or_assign(storePath, storePathForDst); */ if (isValidPath(info.path)) { nrDone++; @@ -1085,10 +1073,32 @@ std::map copyPaths( Store::PathsSource pathsToCopy; + auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath { + auto storePathForSrc = currentPathInfo.path; + auto storePathForDst = storePathForSrc; + if (currentPathInfo.ca && currentPathInfo.references.empty()) { + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); + if (dstStore.storeDir == srcStore.storeDir) + assert(storePathForDst == storePathForSrc); + if (storePathForDst != storePathForSrc) + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePathForSrc), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); + } + return storePathForDst; + }; + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); - auto source = sinkToSource([&](Sink & sink) { + auto storePathForDst = computeStorePathForDst(*info); + pathsMap.insert_or_assign(missingPath, storePathForDst); + + ValidPathInfo infoForDst = *info; + infoForDst.path = storePathForDst; + + auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we // read the path, so log something about that at that point auto srcUri = srcStore.getUri(); @@ -1101,7 +1111,7 @@ std::map copyPaths( srcStore.narFromPath(missingPath, sink); }); - pathsToCopy.push_back(std::pair{*info, std::move(source)}); + pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); From 34d90fbe224e7b626366f98ea1bc9e370d9cb534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 15:25:52 +0200 Subject: [PATCH 04/58] Mention the parallel copy in the release notes --- doc/manual/src/release-notes/rl-next.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 52e3b6240..6c2c4689c 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,3 +2,7 @@ * Nix can now be built with LTO by passing `--enable-lto` to `configure`. LTO is currently only supported when building with GCC. + +* `nix copy` now copies the store paths in parallel as much as possible (again). + This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything + in one batch to avoid latencies issues. From 56f6f3725f4fbeeb7900ae95bd71d559695b3dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 19 Jul 2022 19:46:00 +0200 Subject: [PATCH 05/58] Don't ultimately trust the signed paths Like the old implem did (and like you'd want it to be anyways) --- src/libstore/store-api.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 61a12e84a..06d03bff2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -305,7 +305,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info, source] = *infosMap.at(path); + auto & [info_, source] = *infosMap.at(path); + auto info = info_; + info.ultimate = false; if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); From 1467a98d4c2014e512825c87e9056de79408421a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Mon, 1 Aug 2022 11:37:21 +0200 Subject: [PATCH 06/58] derivation-goal.cc: remove bmCheck custom return branch on buildDone Once a derivation goal has been completed, we check whether or not this goal was meant to be repeated to check its output. An early return branch was preventing the worker to reach that repeat code branch, hence breaking the --check command (#2619). It seems like this early return branch is an artifact of a passed refactoring. As far as I can tell, buildDone's main branch also cleanup the tmp directory before returning. --- src/libstore/build/derivation-goal.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..3c0b14029 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -914,12 +914,6 @@ void DerivationGoal::buildDone() outputPaths ); - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built, std::move(builtOutputs)); - return; - } - cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ From 8119390abcbb25e849acc50d0af0b37d85adfb04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:08 +0100 Subject: [PATCH 07/58] Move some fs-related functions to their own file Unclutter `util.cc` a bit --- src/libutil/filesystem.cc | 44 +++++++++++++++++++++++++++++++++++++++ src/libutil/util.cc | 38 --------------------------------- 2 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 src/libutil/filesystem.cc diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc new file mode 100644 index 000000000..e67383e6f --- /dev/null +++ b/src/libutil/filesystem.cc @@ -0,0 +1,44 @@ +#include + +#include "util.hh" +#include "types.hh" + +namespace nix { + +void createSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } +} + +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp, mtime); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmp, link); + + break; + } +} + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..3f3695f0d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -681,44 +681,6 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } -} - - -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp, mtime); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); - - break; - } -} - - void readFull(int fd, char * buf, size_t count) { while (count) { From c2de0a232c1cfddb1f1385ffd23dd43a2099458e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:46 +0100 Subject: [PATCH 08/58] =?UTF-8?q?Create=20a=20wrapper=20around=20stdlib?= =?UTF-8?q?=E2=80=99s=20`rename`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Directly takes some c++ strings, and gently throws an exception on error (rather than having to inline this logic everywhere) --- src/libstore/build/derivation-goal.cc | 3 +-- src/libstore/build/local-derivation-goal.cc | 8 +++----- src/libstore/builtins/unpack-channel.cc | 3 +-- src/libstore/gc.cc | 4 +--- src/libstore/local-binary-cache-store.cc | 3 +-- src/libstore/local-store.cc | 6 ++---- src/libstore/optimise-store.cc | 6 ++++-- src/libutil/filesystem.cc | 9 +++++++-- src/libutil/util.hh | 2 ++ 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..93f923f18 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 79a241ae0..2435377e2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); + moveFile((chrootRootDir + p), p); } return diskFull; @@ -2625,8 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); + moveFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 426d58a53..a8417d7ff 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) - throw SysError("renaming channel directory"); + moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d58ed78b1..3682f81cc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError("cannot rename '%1%' to '%2%'", - tempLink , link); + moveFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ba4416f6d..ff7403f9d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,8 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - if (rename(tmp.c_str(), path2.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path2); + moveFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eba3b0fa5..7f708b243 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,8 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - if (rename(tempPath.c_str(), realPath.c_str())) - throw Error("renaming '%s' to '%s'", tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1942,8 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - if (rename(tmpFile.c_str(), logPath.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); + moveFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8af9b1dde..20b9c7611 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { + try { + moveFile(tempLink, path); + } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { @@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); + throw; } stats.filesLinked++; diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index e67383e6f..33a8d81a6 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -34,11 +34,16 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); + moveFile(tmp, link); break; } } +void moveFile(const Path & oldName, const Path & newName) +{ + if (::rename(oldName.c_str(), newName.c_str())) + throw SysError("renaming '%1%' to '%2%'", oldName, newName); +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 29227ecc6..564d36e79 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,6 +168,8 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ From 6f89fb60088c4bc1513a005f0350c2bc13068892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 16:13:29 +0100 Subject: [PATCH 09/58] rename: Fallback to a copy if the filesystems mismatch In `nix::rename`, if the call to `rename` fails with `EXDEV` (failure because the source and the destination are in a different filesystems) switch to copying and removing the source. To avoid having to re-implement the copy manually, I switched the function to use the c++17 `filesystem` library (which has a `copy` function that should do what we want). Fix #6262 --- src/libutil/filesystem.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 33a8d81a6..cf99b848a 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,8 +1,11 @@ #include +#include #include "util.hh" #include "types.hh" +namespace fs = std::filesystem; + namespace nix { void createSymlink(const Path & target, const Path & link, @@ -42,8 +45,16 @@ void replaceSymlink(const Path & target, const Path & link, void moveFile(const Path & oldName, const Path & newName) { - if (::rename(oldName.c_str(), newName.c_str())) - throw SysError("renaming '%1%' to '%2%'", oldName, newName); + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + try { + fs::rename(oldPath, newPath); + } catch (fs::filesystem_error & e) { + if (e.code().value() == EXDEV) { + fs::copy(oldName, newName, fs::copy_options::copy_symlinks); + fs::remove_all(oldName); + } + } } } From c5db1821a94406eaff86341220bc301ba1dac82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Mar 2022 14:25:56 +0100 Subject: [PATCH 10/58] Re-implement the recursive directory copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recursive copy from the stl doesn’t exactly do what we need because 1. It doesn’t delete things as we go 2. It doesn’t keep the mtime, which change the nars So re-implement it ourselves. A bit dull, but that way we have what we want --- src/libutil/filesystem.cc | 52 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index cf99b848a..198db2832 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -43,6 +43,53 @@ void replaceSymlink(const Path & target, const Path & link, } } +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + void moveFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); @@ -51,8 +98,9 @@ void moveFile(const Path & oldName, const Path & newName) fs::rename(oldPath, newPath); } catch (fs::filesystem_error & e) { if (e.code().value() == EXDEV) { - fs::copy(oldName, newName, fs::copy_options::copy_symlinks); - fs::remove_all(oldName); + fs::remove(newPath); + warn("Copying %s to %s", oldName, newName); + copy(fs::directory_entry(oldPath), newPath, true); } } } From a4f0fd633c9ec865d385b34bdc51923c204e7ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 11:52:39 +0200 Subject: [PATCH 11/58] Link against c++fs on darwin Required by the old clang version --- src/libutil/local.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..13e8d426a 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,3 +11,7 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif + +ifdef HOST_DARWIN + libutil_LDFLAGS += -lc++fs +endif From d71d9e9fbfc972514b0b940181ab7f9e766f41f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:10:36 +0200 Subject: [PATCH 12/58] moveFile -> renameFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `move` tends to have this `mv` connotation of “I will copy it for you if needs be” --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 +++--- src/libstore/builtins/unpack-channel.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libstore/optimise-store.cc | 2 +- src/libutil/filesystem.cc | 4 ++-- src/libutil/util.hh | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 93f923f18..459fdae79 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,7 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2435377e2..6843173a7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,7 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -310,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - moveFile((chrootRootDir + p), p); + renameFile((chrootRootDir + p), p); } return diskFull; @@ -2624,7 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - moveFile(path, dst); + renameFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index a8417d7ff..ba04bb16c 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,7 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); + renameFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3682f81cc..4c1a82279 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,7 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - moveFile(tempLink, link); + renameFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ff7403f9d..f20b1fa02 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,7 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - moveFile(tmp, path2); + renameFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7f708b243..2d076f404 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - moveFile(tempPath, realPath); + renameFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1941,7 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - moveFile(tmpFile, logPath); + renameFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 20b9c7611..4d2781180 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -230,7 +230,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ try { - moveFile(tempLink, path); + renameFile(tempLink, path); } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 198db2832..fee40d09e 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -37,7 +37,7 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - moveFile(tmp, link); + renameFile(tmp, link); break; } @@ -90,7 +90,7 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) } } -void moveFile(const Path & oldName, const Path & newName) +void renameFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 564d36e79..186513cea 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,7 +168,7 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); -void moveFile(const Path & src, const Path & dst); +void renameFile(const Path & src, const Path & dst); /* Wrappers arount read()/write() that read/write exactly the From 90f968073338d6ba276994bc281fa5efa5306e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:19:42 +0200 Subject: [PATCH 13/58] Only use `renameFile` where needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In most places the fallback to copying isn’t needed and can actually be bad, so we’d rather not transparently fallback --- src/libstore/local-store.cc | 2 +- src/libutil/filesystem.cc | 14 +++++++++----- src/libutil/util.hh | 9 +++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d076f404..a272e4301 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - renameFile(tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index fee40d09e..9cc18507b 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -54,7 +54,6 @@ void setWriteTime(const fs::path & p, const struct stat & st) .tv_sec = st.st_mtime, .tv_usec = 0, }; - warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); if (lutimes(p.c_str(), times) != 0) throw SysError("changing modification time of '%s'", p); } @@ -92,14 +91,19 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) void renameFile(const Path & oldName, const Path & newName) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ try { - fs::rename(oldPath, newPath); + renameFile(oldName, newName); } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); if (e.code().value() == EXDEV) { fs::remove(newPath); - warn("Copying %s to %s", oldName, newName); + warn("Can’t rename %s as %s, copying instead", oldName, newName); copy(fs::directory_entry(oldPath), newPath, true); } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 186513cea..cd83f250f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -170,6 +170,15 @@ void replaceSymlink(const Path & target, const Path & link, void renameFile(const Path & src, const Path & dst); +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ From 1ba5b3e001c3da3c2e2a4dc7d475da1b20f53638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:48:31 +0200 Subject: [PATCH 14/58] Make `moveFile` more atomic Rather than directly copying the source to its dest, copy it first to a temporary location, and eventually move that temporary. That way, the move is at least atomic from the point-of-view of the destination --- src/libutil/filesystem.cc | 62 ++++++++++++++++++++++++++++++++++++++- src/libutil/util.cc | 55 ---------------------------------- 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 9cc18507b..403389e60 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,6 +1,7 @@ #include #include +#include "finally.hh" #include "util.hh" #include "types.hh" @@ -8,6 +9,59 @@ namespace fs = std::filesystem; namespace nix { +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + void createSymlink(const Path & target, const Path & link, std::optional mtime) { @@ -101,10 +155,16 @@ void moveFile(const Path & oldName, const Path & newName) } catch (fs::filesystem_error & e) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { fs::remove(newPath); warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), newPath, true); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); } } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3f3695f0d..2a1bb770a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -508,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed) } -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); From d1cda07ce47064bda2c609a0290c867295ddd0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 25 May 2022 11:37:01 +0200 Subject: [PATCH 15/58] Don't use -load_all on darwin That flag breaks `-lc++fs` (introducing a duplicate symbol for some reason). Besides, it was apparently needed for bzip2, but we're not using bzip2 anymore. --- configure.ac | 9 --------- 1 file changed, 9 deletions(-) diff --git a/configure.ac b/configure.ac index f0210ab78..64fa12fc7 100644 --- a/configure.ac +++ b/configure.ac @@ -296,15 +296,6 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) -# This is needed if bzip2 is a static library, and the Nix libraries -# are dynamic. -case "${host_os}" in - darwin*) - LDFLAGS="-all_load $LDFLAGS" - ;; -esac - - AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) From b47b6a418d656ee5408b7773dd9b76770ca0d2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Wed, 3 Aug 2022 12:05:41 +0200 Subject: [PATCH 16/58] tests/check.sh: add nix3-build check test --- tests/check.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/check.sh b/tests/check.sh index ab48ff865..495202781 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -40,6 +40,14 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix deterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log +if grep -q 'checking is not possible' $TEST_ROOT/log; then false; fi +# Repeat is set to 1, ie. nix should build deterministic twice. +if [ "$(grep "checking outputs" $TEST_ROOT/log | wc -l)" -ne 2 ]; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link 2> $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log @@ -50,6 +58,12 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix nondeterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log From cb6794a0d983eb364601f26fc32ead98ed67bfb4 Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Sun, 7 Aug 2022 10:13:11 -0400 Subject: [PATCH 17/58] Do not spam logs if the owned-homedir check results in a noop --- src/libutil/util.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..e11cb9c60 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -577,6 +577,7 @@ Path getHome() { static Path homeDir = []() { + std::optional unownedUserHomeDir = {}; auto homeDir = getEnv("HOME"); if (homeDir) { // Only use $HOME if doesn't exist or is owned by the current user. @@ -588,8 +589,7 @@ Path getHome() homeDir.reset(); } } else if (st.st_uid != geteuid()) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir); - homeDir.reset(); + unownedUserHomeDir.swap(homeDir); } } if (!homeDir) { @@ -600,6 +600,9 @@ Path getHome() || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } } return *homeDir; }(); From 96b4339d86a22175da22900acecd6b0fb4d5e12b Mon Sep 17 00:00:00 2001 From: Tom Franklin Date: Mon, 8 Aug 2022 00:42:18 +0100 Subject: [PATCH 18/58] Only fail if selinux is in enforcing --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a18280ef..e6864eaaf 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -640,7 +640,7 @@ place_channel_configuration() { check_selinux() { if command -v getenforce > /dev/null 2>&1; then - if ! [ "$(getenforce)" = "Disabled" ]; then + if [ "$(getenforce)" = "Enforcing" ]; then failure < Date: Mon, 8 Aug 2022 14:34:22 +0200 Subject: [PATCH 19/58] Remove the explicit `c++fs` linkage on darwin Doesn't seem needed on a recent-enough clang anymore (and even seems to break stuff) --- src/libutil/local.mk | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 13e8d426a..f880c0fc5 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,7 +11,3 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif - -ifdef HOST_DARWIN - libutil_LDFLAGS += -lc++fs -endif From 5f37c5191a3a8f5c7ab31a0dd8bffe14aaa6b76c Mon Sep 17 00:00:00 2001 From: Winter Date: Tue, 9 Aug 2022 16:48:34 -0400 Subject: [PATCH 20/58] nix-shell: specify which outputs from bashInteractive to build --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 7eb8c8f6a..df292dce6 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -401,7 +401,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { .drvPath = bashDrv, - .outputs = {}, + .outputs = {"out"}, }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; From 0eb9946e1d3621cfc2fcffc9378dba334b25fb26 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 9 Aug 2022 23:21:09 -0400 Subject: [PATCH 21/58] docker.nix: Provide boolean for whether to bundle nixpkgs --- docker.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index 8e6aa227f..e95caf274 100644 --- a/docker.nix +++ b/docker.nix @@ -2,6 +2,7 @@ , lib ? pkgs.lib , name ? "nix" , tag ? "latest" +, bundleNixpkgs ? true , channelName ? "nixpkgs" , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" , extraPkgs ? [] @@ -139,10 +140,12 @@ let baseSystem = let nixpkgs = pkgs.path; - channel = pkgs.runCommand "channel-nixos" { } '' + channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } '' mkdir $out - ln -s ${nixpkgs} $out/nixpkgs - echo "[]" > $out/manifest.nix + if [ "$bundleNixpkgs" ]; then + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + fi ''; rootEnv = pkgs.buildPackages.buildEnv { name = "root-profile-env"; From 64c3adbe1ad14e043b772c3e981d511922cb06e5 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 8 Aug 2022 15:45:23 -0400 Subject: [PATCH 22/58] install-multi-user: abstract is_root, is_os_linux, is_os_darwin --- scripts/install-multi-user.sh | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a18280ef..472d25842 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -59,6 +59,30 @@ headless() { fi } +is_root() { + if [ "$EUID" -eq 0 ]; then + return 0 + else + return 1 + fi +} + +is_os_linux() { + if [ "$(uname -s)" = "Linux" ]; then + return 0 + else + return 1 + fi +} + +is_os_darwin() { + if [ "$(uname -s)" = "Darwin" ]; then + return 0 + else + return 1 + fi +} + contact_us() { echo "You can open an issue at https://github.com/nixos/nix/issues" echo "" @@ -423,7 +447,7 @@ EOF fi done - if [ "$(uname -s)" = "Linux" ] && [ ! -e /run/systemd/system ]; then + if is_os_linux && [ ! -e /run/systemd/system ]; then warning < Date: Mon, 8 Aug 2022 16:01:59 -0400 Subject: [PATCH 23/58] Strip whitespace in installing-binary.md --- doc/manual/src/installation/installing-binary.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 9fb9c80c3..18fc8fff5 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -13,7 +13,7 @@ for your platform: - multi-user on macOS > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > + > > - It took some time to support this cleanly. You may see posts, > examples, and tutorials using obsolete workarounds. > - Supporting it cleanly made macOS installs too complex to qualify @@ -75,7 +75,7 @@ should run this under your usual user account, *not* as root. The script will invoke `sudo` as needed. > **Note** -> +> > If you need Nix to use a different group ID or user ID set, you will > have to download the tarball manually and [edit the install > script](#installing-from-a-binary-tarball). @@ -167,7 +167,7 @@ and `/etc/zshrc` which you may remove. removed next. 7. Remove the Nix Store volume: - + ```console sudo diskutil apfs deleteVolume /nix ``` @@ -176,7 +176,7 @@ and `/etc/zshrc` which you may remove. store. > **Note** -> +> > After you complete the steps here, you will still have an empty `/nix` > directory. This is an expected sign of a successful uninstall. The empty > `/nix` directory will disappear the next time you reboot. From 7bb1e913b33499d3ce74929749977774bcc35aed Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 8 Aug 2022 15:46:17 -0400 Subject: [PATCH 24/58] Don't prompt about using sudo if we're already root --- .../src/installation/installing-binary.md | 6 ++--- scripts/install-multi-user.sh | 25 ++++++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 18fc8fff5..a46735196 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -31,8 +31,8 @@ $ sh <(curl -L https://nixos.org/nix/install) --no-daemon ``` This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You should run this under your usual user -account, *not* as root. The script will invoke `sudo` to create `/nix` +is owned by the invoking user. You can run this under your usual user +account or root. The script will invoke `sudo` to create `/nix` if it doesn’t already exist. If you don’t have `sudo`, you should manually create `/nix` first as root, e.g.: @@ -71,7 +71,7 @@ $ sh <(curl -L https://nixos.org/nix/install) --daemon The multi-user installation of Nix will create build users between the user IDs 30001 and 30032, and a group with the group ID 30000. You -should run this under your usual user account, *not* as root. The script +can run this under your usual user account or root. The script will invoke `sudo` as needed. > **Note** diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 472d25842..e7bdc1227 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -337,10 +337,15 @@ __sudo() { _sudo() { local expl="$1" shift - if ! headless; then + if ! headless || is_root; then __sudo "$expl" "$*" >&2 fi - sudo "$@" + + if is_root; then + env "$@" + else + sudo "$@" + fi } @@ -891,17 +896,6 @@ EOF main() { - # TODO: I've moved this out of validate_starting_assumptions so we - # can fail faster in this case. Sourcing install-darwin... now runs - # `touch /` to detect Read-only root, but it could update times on - # pre-Catalina macOS if run as root user. - if is_root; then - failure < Date: Wed, 10 Aug 2022 18:49:29 -0500 Subject: [PATCH 25/58] docfix: bundlers --- src/nix/bundle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/bundle.md b/src/nix/bundle.md index 2bb70711f..a18161a3c 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -44,7 +44,7 @@ flake output attributes: * `bundlers..default` -If an attribute *name* is given, `nix run` tries the following flake +If an attribute *name* is given, `nix bundle` tries the following flake output attributes: * `bundlers..` From 703b335c1d187fb18ca5a5450b33e4e9a98abc7c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 15 Aug 2022 11:34:39 +0200 Subject: [PATCH 26/58] reword description of language properties the list style is supposed to give more structure. each property is explained as concisely as possible while trying not to sound too fancy. --- doc/manual/src/language/index.md | 38 ++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index c4b3abf75..a4b402f8b 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,13 +1,33 @@ # Nix Language -The Nix language is a pure, lazy, functional language. Purity -means that operations in the language don't have side-effects (for -instance, there is no variable assignment). Laziness means that -arguments to functions are evaluated only when they are needed. -Functional means that functions are “normal” values that can be passed -around and manipulated in interesting ways. The language is not a -full-featured, general purpose language. Its main job is to describe -packages, compositions of packages, and the variability within packages. +The Nix language is -This section presents the various features of the language. +- *domain-specific* + + It only exists for the Nix package manager: + to describe packages and configurations as well as their variants and compositions. + It is not intended for general purpose use. + +- *declarative* + + There is no notion of executing sequential steps. + Dependencies between operations are established only through data. + +- *pure* + + Values cannot change during computation. + Functions always produce the same output if their input does not change. + +- *functional* + + Functions are like any other value. + Functions can be assigned to names, taken as arguments, or returned by functions. + +- *lazy* + + Expressions are only evaluated when their value is needed. + +- *dynamically typed* + + Type errors are only detected when expressions are evaluated. From 3d4489b623deaceef720da7d884a41452f928db6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 15:57:25 +0200 Subject: [PATCH 27/58] Show when we're evaluating a flake --- src/libcmd/installables.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 59162c4df..e097f23b3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -616,6 +616,8 @@ InstallableFlake::InstallableFlake( std::tuple InstallableFlake::toDerivation() { + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + auto attr = getCursor(*state); auto attrPath = attr->getAttrPathStr(); From e62160579f40a0425061c2223e0a303d42736ea2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Aug 2022 14:58:08 +0200 Subject: [PATCH 28/58] nix flake metadata: Don't show "Inputs" if there are no inputs --- src/nix/flake.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e01bc6d10..3967f1102 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -212,7 +212,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + if (!lockedFlake.lockFile.root->inputs.empty()) + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); std::unordered_set> visited; From c3769c68465bae971ab6bb48cfcdea85b61ea83a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 15:56:08 +0200 Subject: [PATCH 29/58] ProgressBar: Delay before showing a new activity Some activities are numerous but usually very short (e.g. copying a source file to the store) which would cause a lot of flickering. So only show activities that have been running for at least 10 ms. --- src/libmain/progress-bar.cc | 43 ++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f4306ab91..5183f212f 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -8,6 +8,7 @@ #include #include #include +#include namespace nix { @@ -48,6 +49,7 @@ private: bool visible = true; ActivityId parent; std::optional name; + std::chrono::time_point startTime; }; struct ActivitiesByType @@ -91,10 +93,11 @@ public: state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); + auto nextWakeup = std::chrono::milliseconds::max(); while (state->active) { if (!state->haveUpdate) - state.wait(updateCV); - draw(*state); + state.wait_for(updateCV, nextWakeup); + nextWakeup = draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } }); @@ -118,7 +121,8 @@ public: updateThread.join(); } - bool isVerbose() override { + bool isVerbose() override + { return printBuildLogs; } @@ -159,11 +163,13 @@ public: if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) log(*state, lvl, s + "..."); - state->activities.emplace_back(ActInfo()); + state->activities.emplace_back(ActInfo { + .s = s, + .type = type, + .parent = parent, + .startTime = std::chrono::steady_clock::now() + }); auto i = std::prev(state->activities.end()); - i->s = s; - i->type = type; - i->parent = parent; state->its.emplace(act, i); state->activitiesByType[type].its.emplace(act, i); @@ -327,10 +333,12 @@ public: updateCV.notify_one(); } - void draw(State & state) + std::chrono::milliseconds draw(State & state) { + auto nextWakeup = std::chrono::milliseconds::max(); + state.haveUpdate = false; - if (!state.active) return; + if (!state.active) return nextWakeup; std::string line; @@ -341,12 +349,25 @@ public: line += "]"; } + auto now = std::chrono::steady_clock::now(); + if (!state.activities.empty()) { if (!status.empty()) line += " "; auto i = state.activities.rbegin(); - while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) + while (i != state.activities.rend()) { + if (i->visible && (!i->s.empty() || !i->lastLine.empty())) { + /* Don't show activities until some time has + passed, to avoid displaying very short + activities. */ + auto delay = std::chrono::milliseconds(10); + if (i->startTime + delay < now) + break; + else + nextWakeup = std::min(nextWakeup, std::chrono::duration_cast(delay - (now - i->startTime))); + } ++i; + } if (i != state.activities.rend()) { line += i->s; @@ -366,6 +387,8 @@ public: if (width <= 0) width = std::numeric_limits::max(); writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + + return nextWakeup; } std::string getStatus(State & state) From 53e7b7e8ac44704199a868c0f6850ac53a0b8ad1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 12:28:02 +0200 Subject: [PATCH 30/58] Remove warnLargeDump() This message was unhelpful (#1184) and probably misleading since memory is O(1) in most cases now. --- src/libstore/remote-store.cc | 2 -- src/libutil/serialise.cc | 20 -------------------- src/libutil/serialise.hh | 4 +--- 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..eaaf9669f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -580,7 +580,6 @@ ref RemoteStore::addCAToStore( try { conn->to.written = 0; - conn->to.warn = true; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); @@ -591,7 +590,6 @@ ref RemoteStore::addCAToStore( dumpString(contents, conn->to); } } - conn->to.warn = false; conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8ff904583..2c3597775 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -48,24 +48,9 @@ FdSink::~FdSink() } -size_t threshold = 256 * 1024 * 1024; - -static void warnLargeDump() -{ - warn("dumping very large path (> 256 MiB); this may run out of memory"); -} - - void FdSink::write(std::string_view data) { written += data.size(); - static bool warned = false; - if (warn && !warned) { - if (written > threshold) { - warnLargeDump(); - warned = true; - } - } try { writeFull(fd, data); } catch (SysError & e) { @@ -448,11 +433,6 @@ Error readError(Source & source) void StringSink::operator () (std::string_view data) { - static bool warned = false; - if (!warned && s.size() > threshold) { - warnLargeDump(); - warned = true; - } s.append(data); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 13da26c6a..84847835a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -97,19 +97,17 @@ protected: struct FdSink : BufferedSink { int fd; - bool warn = false; size_t written = 0; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink && s) + FdSink & operator=(FdSink && s) { flush(); fd = s.fd; s.fd = -1; - warn = s.warn; written = s.written; return *this; } From 81e42e0d3f0345c28f3d19841c89c4b1975c37a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jul 2022 16:41:26 +0200 Subject: [PATCH 31/58] Fix onError --- tests/common.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.sh.in b/tests/common.sh.in index 79da10199..73c2d2309 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -193,7 +193,7 @@ fi onError() { set +x echo "$0: test failed at:" >&2 - for ((i = 1; i < 16; i++)); do + for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 done From c80a74b7d52f0cf5fc5c1c7497e059ebf301f789 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 16:59:02 +0200 Subject: [PATCH 32/58] Don't pass --force to 'git add' Fixes #5810. --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d01aaa7a..c1a21e764 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, From 6547dcde2a37179f98cc9a8702bc15a2fd1e6a4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:41:19 +0200 Subject: [PATCH 33/58] Use plain mktemp This fixes the case where $TMPDIR doesn't end in a slash. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a9f3e74dc..7c8e159b5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,7 +349,7 @@ _sudo() { } -readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" } From 823e1017d809219e7e00b24ef4ccb0a8b568449c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:47:01 +0200 Subject: [PATCH 34/58] Ensure that $TMPDIR exists if defined --- scripts/install-multi-user.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7c8e159b5..1431857d5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,6 +349,11 @@ _sudo() { } +# Ensure that $TMPDIR exists if defined. +if [[ -v TMPDIR ]]; then + mkdir -m 0700 -p "$TMPDIR" +fi + readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" From 8188b1d0abc2eba6497b5dc47f7e848cbacb7677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:59:04 +0200 Subject: [PATCH 35/58] json: write null on abnormal placeholder destruction Avoids leaving dangling attributes like { "foo": } in case of exceptions. --- src/libutil/json.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libutil/json.cc b/src/libutil/json.cc index abe0e6e74..2f9e97ff5 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -193,7 +193,11 @@ JSONObject JSONPlaceholder::object() JSONPlaceholder::~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + if (first) { + assert(std::uncaught_exceptions()); + if (state->stack != 0) + write(nullptr); + } } } From 7535ee345da6c0aea1dd81e7de7725b37d8bf8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:33:46 +0200 Subject: [PATCH 36/58] nix-env: don't output incomplete JSON --- src/nix-env/nix-env.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..fdd66220a 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -940,12 +940,12 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin JSONObject metaObj = pkgObj.object("meta"); StringSet metaNames = i.queryMetaNames(); for (auto & j : metaNames) { - auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - placeholder.write(nullptr); + metaObj.attr(j, nullptr); } else { + auto placeholder = metaObj.placeholder(j); PathSet context; printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } From 7d934f7880d460cba1ab908fd35b3d43e97a4749 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 19 Aug 2022 11:26:26 +0200 Subject: [PATCH 37/58] don't read outputs into memory for output rewriting RewritingSink can handle being fed input where a reference crosses a chunk boundary. we don't need to load the whole source into memory, and in fact *not* loading the whole source lets nix build FODs that do not fit into memory (eg fetchurl'ing data files larger than system memory). --- src/libstore/build/local-derivation-goal.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6843173a7..18b682e13 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2374,10 +2374,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() if (*scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); - rsink2(sink.s); + dumpPath(actualPath, rsink2); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; From 0d2bf7acf994ba331d6f72c746721b354931be76 Mon Sep 17 00:00:00 2001 From: Solene Rapenne Date: Fri, 19 Aug 2022 12:40:22 +0200 Subject: [PATCH 38/58] add a nix.conf option to set a download speed limit --- src/libstore/filetransfer.cc | 3 +++ src/libstore/globals.hh | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8454ad7d2..252403cb5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + if (settings.downloadSpeed.get() > 0) + curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..ca8fc6d5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -746,6 +746,13 @@ public: /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. )"}; + Setting downloadSpeed { + this, 0, "download-speed", + R"( + Specify the maxium transfer rate in kilobytes per second you want + nix to use for download. + )"}; + Setting netrcFile{ this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", R"( From 0bf52b73f4cb61bc12c95a015a7be45f7174ca01 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 19 Aug 2022 15:03:37 -0500 Subject: [PATCH 39/58] install: only create TMPDIR if missing --- scripts/install-multi-user.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 1431857d5..01dbf0c0e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -348,10 +348,9 @@ _sudo() { fi } - # Ensure that $TMPDIR exists if defined. -if [[ -v TMPDIR ]]; then - mkdir -m 0700 -p "$TMPDIR" +if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then + mkdir -m 0700 -p "${TMPDIR:-}" fi readonly SCRATCH=$(mktemp -d) From 7d800909e94c482a2093bc95a2f3dca565c148b2 Mon Sep 17 00:00:00 2001 From: Jakub Kuczys Date: Sat, 20 Aug 2022 03:48:42 +0200 Subject: [PATCH 40/58] Fix default profile path for root in nix profile documentation --- src/nix/profile.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.md b/src/nix/profile.md index 8dade051d..be3c5ba1a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. The default profile used by `nix profile` is `$HOME/.nix-profile`, which, if it does not exist, is created as a symlink to -`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`/nix/var/nix/profiles/default` if Nix is invoked by the `root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. You can specify another profile location using `--profile` *path*. From caad87e6dbfbc62fafb4a055b45d7f2eb3d11efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Sat, 20 Aug 2022 18:21:36 +0200 Subject: [PATCH 41/58] Better documentation wording Co-authored-by: Anderson Torres --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca8fc6d5f..1ff7d2d0a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -749,7 +749,7 @@ public: Setting downloadSpeed { this, 0, "download-speed", R"( - Specify the maxium transfer rate in kilobytes per second you want + Specify the maximum transfer rate in kilobytes per second you want nix to use for download. )"}; From c21b1a7e67cc28b6e95a563daa786f385bc716b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:14:14 +0200 Subject: [PATCH 42/58] Spelling --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1ff7d2d0a..e9d721e59 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -750,7 +750,7 @@ public: this, 0, "download-speed", R"( Specify the maximum transfer rate in kilobytes per second you want - nix to use for download. + Nix to use for downloads. )"}; Setting netrcFile{ From 0d2163c6dcf03463fa91ec6d0d96c928ad907366 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:27:36 +0200 Subject: [PATCH 43/58] nix repl: Stop the progress bar The repl was broken since c3769c68465bae971ab6bb48cfcdea85b61ea83a. In general, the progress bar is incompatible with the repl. --- src/libcmd/repl.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 23df40337..150bd42ac 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -35,6 +35,7 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" +#include "progress-bar.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -252,6 +253,10 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif + /* Stop the progress bar because it interferes with the display of + the repl. */ + stopProgressBar(); + std::string input; while (true) { @@ -1037,9 +1042,10 @@ void runRepl( struct CmdRepl : InstallablesCommand { - CmdRepl(){ + CmdRepl() { evalSettings.pureEval = false; } + void prepare() { if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { @@ -1053,12 +1059,15 @@ struct CmdRepl : InstallablesCommand } installables = InstallablesCommand::load(); } + std::vector files; + Strings getDefaultFlakeAttrPaths() override { return {""}; } - virtual bool useDefaultInstallables() override + + bool useDefaultInstallables() override { return file.has_value() or expr.has_value(); } From 062e4fcdde145ec6780df8d1002dc7380f6eb4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 16 Aug 2022 12:23:37 +0200 Subject: [PATCH 44/58] JSON: print paths as strings without copying them to the store Makes `printValueAsJSON` not copy paths to the store for `nix eval --json`, `nix-instantiate --eval --json` and `nix-env --json`. Fixes https://github.com/NixOS/nix/issues/5612 --- src/libexpr/value-to-json.cc | 21 ++++++++++++--------- src/libexpr/value-to-json.hh | 4 ++-- src/libexpr/value.hh | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/eval.cc | 2 +- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 03504db61..4d63d8b49 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,7 +10,7 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context) + Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore) { checkInterrupt(); @@ -32,7 +32,10 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nPath: - out.write(state.copyPathToStore(context, v.path)); + if (copyToStore) + out.write(state.copyPathToStore(context, v.path)); + else + out.write(v.path); break; case nNull: @@ -54,10 +57,10 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context); + printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore); } } else - printValueAsJSON(state, strict, *i->value, i->pos, out, context); + printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore); break; } @@ -65,13 +68,13 @@ void printValueAsJSON(EvalState & state, bool strict, auto list(out.list()); for (auto elem : v.listItems()) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *elem, pos, placeholder, context); + printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore); } break; } case nExternal: - v.external->printValueAsJSON(state, strict, out, context); + v.external->printValueAsJSON(state, strict, out, context, copyToStore); break; case nFloat: @@ -91,14 +94,14 @@ void printValueAsJSON(EvalState & state, bool strict, } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context) + Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, pos, out, context); + printValueAsJSON(state, strict, v, pos, out, context, copyToStore); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const + JSONPlaceholder & out, PathSet & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index c020a817a..7ddc8a5b1 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context); + Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context); + Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 2008df74d..590ba7783 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -99,7 +99,7 @@ class ExternalValueBase /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual void printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const; + JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const; /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..c4e981d49 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -947,7 +947,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context, false); } } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index d3144e131..6181d2190 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -53,7 +53,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); else { if (strict) state.forceValueDeep(vRes); vRes.print(state.symbols, std::cout); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 967dc8519..cc022ae43 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -116,7 +116,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, pos, jsonOut, context); + printValueAsJSON(*state, true, *v, pos, jsonOut, context, false); } else { From 4c2ff4a0f4c2106d5792a87a1ba9ee1fd18c0e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 22 Aug 2022 15:07:52 +0200 Subject: [PATCH 45/58] JSON: add missing newlines after `nix eval --json` and `nix-instantiate --eval --json`. --- src/nix-instantiate/nix-instantiate.cc | 5 +++-- src/nix/eval.cc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6181d2190..80f35828c 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -52,9 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths, state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); - else if (output == okJSON) + else if (output == okJSON) { printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); - else { + std::cout << std::endl; + } else { if (strict) state.forceValueDeep(vRes); vRes.print(state.symbols, std::cout); std::cout << std::endl; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index cc022ae43..ddd2790c6 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -117,6 +117,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, pos, jsonOut, context, false); + std::cout << std::endl; } else { From f865048332a26f69a007881877203b9428783357 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 15:30:38 +0200 Subject: [PATCH 46/58] Indentation --- src/libstore/remote-store.hh | 10 ++++------ src/libstore/store-api.hh | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 5a599997e..11d089cd2 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -89,12 +89,10 @@ public: CheckSigsFlag checkSigs) override; void addMultipleToStore( - PathsSource & pathsToCopy, - Activity & act, - RepairFlag repair, - CheckSigsFlag checkSigs) override; - - + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c0a61115b..c8a667c6d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -369,12 +369,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, - CheckSigsFlag checkSigs = CheckSigs - ); + CheckSigsFlag checkSigs = CheckSigs); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. From f0358ed4650e4608a383bd9f59ee23545f86c4ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Aug 2022 14:14:47 +0200 Subject: [PATCH 47/58] Fix a hang in nix-copy-ssh.sh This hang for some reason didn't trigger in the Nix build, but did running 'make installcheck' interactively. What happened: * Store::addMultipleToStore() calls a SinkToSource object to copy a path, which in turn calls LegacySSHStore::narFromPath(), which acquires a connection. * The SinkToSource object is not destroyed after the last bytes has been read, so the coroutine's stack is still alive and its destructors are not run. So the connection is not released. * Then when the next path is copied, because max-connections = 1, LegacySSHStore::narFromPath() hangs forever waiting for a connection to be released. The fix is to make sure that the source object is destroyed when we're done with it. --- src/libstore/store-api.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1406bf657..9c3a0b3d6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,6 +11,7 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include "finally.hh" #include @@ -271,7 +272,7 @@ void Store::addMultipleToStore( using PathWithInfo = std::pair>; - std::map infosMap; + std::map infosMap; StorePathSet storePathsToAdd; for (auto & thingToAdd : pathsToCopy) { infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); @@ -288,7 +289,8 @@ void Store::addMultipleToStore( storePathsToAdd, [&](const StorePath & path) { - auto & [info, source] = *infosMap.at(path); + + auto & [info, _] = *infosMap.at(path); if (isValidPath(info.path)) { nrDone++; @@ -309,12 +311,21 @@ void Store::addMultipleToStore( auto info = info_; info.ultimate = false; + /* Make sure that the Source object is destroyed when + we're done. In particular, a SinkToSource object must + be destroyed to ensure that the destructors on its + stack frame are run; this includes + LegacySSHStore::narFromPath()'s connection lock. */ + Finally cleanupSource{[&]() { + source.reset(); + }}; + if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); showProgress(); try { addToStore(info, *source, repair, checkSigs); - } catch (Error &e) { + } catch (Error & e) { nrFailed++; if (!settings.keepGoing) throw e; From ff0b5a778c41a94075d7c651477c4a3a8b4b00cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 23 Aug 2022 14:40:27 +0200 Subject: [PATCH 48/58] Revert to `copyStore = true` for `nix-instantiate` and `nix-env` --- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c4e981d49..a69d3700d 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -947,7 +947,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context, false); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 80f35828c..6b5ba595d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -53,7 +53,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) { - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); std::cout << std::endl; } else { if (strict) state.forceValueDeep(vRes); From db026103b18fb8b5a719594502edd0f89eb9c268 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Aug 2022 14:57:08 +0200 Subject: [PATCH 49/58] nix develop: Ignore some more bash special variables Fixes #6940. --- src/nix/get-env.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 42c806450..a7a8a01b9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -43,6 +43,7 @@ __dumpEnv() { local __var_name="${BASH_REMATCH[2]}" if [[ $__var_name =~ ^BASH_ || \ + $__var_name =~ ^COMP_ || \ $__var_name = _ || \ $__var_name = DIRSTACK || \ $__var_name = EUID || \ @@ -54,7 +55,9 @@ __dumpEnv() { $__var_name = PWD || \ $__var_name = RANDOM || \ $__var_name = SHLVL || \ - $__var_name = SECONDS \ + $__var_name = SECONDS || \ + $__var_name = EPOCHREALTIME || \ + $__var_name = EPOCHSECONDS \ ]]; then continue; fi if [[ -z $__first ]]; then printf ',\n'; else __first=; fi From 4d4f2d10e7ffceac7a700abe0004ffb5b2b23b33 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Tue, 23 Aug 2022 19:38:53 -0400 Subject: [PATCH 50/58] darwin-install: fix shell hint --- scripts/install-darwin-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index afaa6783b..5111a5dde 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -167,7 +167,7 @@ poly_user_shell_get() { } poly_user_shell_set() { - _sudo "in order to give $1 a safe home directory" \ + _sudo "in order to give $1 a safe shell" \ /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" } From c2d74569269d9642448a701aa6efe08ec0fe6f00 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 24 Aug 2022 01:54:43 +0200 Subject: [PATCH 51/58] Fix a misplaced parenthese in serve protocol check This issue made it impossible for clients using a serve protocol of version <= 2.3 to use the `cmdBuildDerivation` command of servers using a protocol of version >= 2.6. The faulty version check makes the server send back build outputs that the client is not expecting. --- src/nix-store/nix-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b453ea1ca..23f2ad3cf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -922,7 +922,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion >= 6)) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { worker_proto::write(*store, out, status.builtOutputs); } From 8d906b1f3bd4343b6b309ddfca824d5dd00a09b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:11:03 +0200 Subject: [PATCH 52/58] Fix macOS build --- src/libstore/store-api.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9c3a0b3d6..2cd6c15ec 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -307,8 +307,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info_, source] = *infosMap.at(path); + auto & [info_, source_] = *infosMap.at(path); auto info = info_; + auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when From 56d97d4b4df02e3464a2f003a90b7f6abae16722 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:49:58 +0200 Subject: [PATCH 53/58] Remove redundant Finally --- src/libstore/store-api.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2cd6c15ec..86b12257a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,7 +11,6 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" -#include "finally.hh" #include @@ -309,7 +308,6 @@ void Store::addMultipleToStore( auto & [info_, source_] = *infosMap.at(path); auto info = info_; - auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when @@ -317,9 +315,7 @@ void Store::addMultipleToStore( be destroyed to ensure that the destructors on its stack frame are run; this includes LegacySSHStore::narFromPath()'s connection lock. */ - Finally cleanupSource{[&]() { - source.reset(); - }}; + auto source = std::move(source_); if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); From a17ce0a8a9c3dd96d0a5043a80acab0c6810a199 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 21:19:43 +0200 Subject: [PATCH 54/58] Fix evaluation cache 98e361ad4c1a26d4ffe4762a6f33bb9e39321a39 introduced a regression where previously stored attributes were replaced by placeholders. As a result, a command like 'nix build nixpkgs#hello' had to be executed at least twice to get caching. This code does not seem necessary for suggestions to work. --- src/libexpr/eval-cache.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0d83b6cfe..b259eec63 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -507,11 +507,6 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - for (auto & attr : *v.attrs) { - if (root->db) - root->db->setPlaceholder({cachedValue->first, attr.name}); - } - auto attr = v.attrs->get(name); if (!attr) { From bb411e4ae16d6a5c61ea595c0c12e2ecee081ff9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:36:40 +0200 Subject: [PATCH 55/58] Fix progress bar flicker with -L This was caused by -L calling setLogFormat() again, which caused the creation of a new progress bar without destroying the old one. So we had two progress bars clobbering each other. We should change 'logger' to be a smart pointer, but I'll do that in a future PR. Fixes #6931. --- src/libmain/loggers.cc | 7 +++++-- src/libmain/progress-bar.cc | 23 ++++++++++++----------- src/libmain/progress-bar.hh | 4 ++-- src/libutil/logging.hh | 3 +++ src/nix/main.cc | 2 +- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cdf23859b..cda5cb939 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -30,8 +30,11 @@ Logger * makeDefaultLogger() { return makeJSONLogger(*makeSimpleLogger(true)); case LogFormat::bar: return makeProgressBar(); - case LogFormat::barWithLogs: - return makeProgressBar(true); + case LogFormat::barWithLogs: { + auto logger = makeProgressBar(); + logger->setPrintBuildLogs(true); + return logger; + } default: abort(); } diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 5183f212f..0bbeaff8d 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -81,14 +81,13 @@ private: std::condition_variable quitCV, updateCV; - bool printBuildLogs; + bool printBuildLogs = false; bool isTTY; public: - ProgressBar(bool printBuildLogs, bool isTTY) - : printBuildLogs(printBuildLogs) - , isTTY(isTTY) + ProgressBar(bool isTTY) + : isTTY(isTTY) { state_.lock()->active = isTTY; updateThread = std::thread([&]() { @@ -503,19 +502,21 @@ public: draw(*state); return s[0]; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { + this->printBuildLogs = printBuildLogs; + } }; -Logger * makeProgressBar(bool printBuildLogs) +Logger * makeProgressBar() { - return new ProgressBar( - printBuildLogs, - shouldANSI() - ); + return new ProgressBar(shouldANSI()); } -void startProgressBar(bool printBuildLogs) +void startProgressBar() { - logger = makeProgressBar(printBuildLogs); + logger = makeProgressBar(); } void stopProgressBar() diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 7f0dafecf..3a76f8448 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -4,9 +4,9 @@ namespace nix { -Logger * makeProgressBar(bool printBuildLogs = false); +Logger * makeProgressBar(); -void startProgressBar(bool printBuildLogs = false); +void startProgressBar(); void stopProgressBar(); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 6f81b92de..d0817b4a9 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -111,6 +111,9 @@ public: virtual std::optional ask(std::string_view s) { return {}; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { } }; ActivityId getCurActivity(); diff --git a/src/nix/main.cc b/src/nix/main.cc index a8404a2ea..f434e9655 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -82,7 +82,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .shortName = 'L', .description = "Print full build logs on standard error.", .category = loggingCategory, - .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, + .handler = {[&]() { logger->setPrintBuildLogs(true); }}, }); addFlag({ From d046eb1463fad967d47cc5becfe5b7a08b5adfa8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:42:34 +0200 Subject: [PATCH 56/58] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index f161b5d80..ed0edc885 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.10.0 \ No newline at end of file +2.11.0 \ No newline at end of file From b0488a29dc7401e5ecd9221215da5ea9879e56d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:44:58 +0200 Subject: [PATCH 57/58] Branch 2.11 release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.11.md | 5 +++++ doc/manual/src/release-notes/rl-next.md | 3 --- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.11.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8fbb59716..9b66ec3db 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -73,6 +73,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) diff --git a/doc/manual/src/release-notes/rl-2.11.md b/doc/manual/src/release-notes/rl-2.11.md new file mode 100644 index 000000000..b322a4e5e --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.11.md @@ -0,0 +1,5 @@ +# Release 2.11 (2022-08-24) + +* `nix copy` now copies the store paths in parallel as much as possible (again). + This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything + in one batch to avoid latencies issues. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7d82c3dc4..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,2 @@ # Release X.Y (202?-??-??) -* `nix copy` now copies the store paths in parallel as much as possible (again). - This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything - in one batch to avoid latencies issues. \ No newline at end of file From 57cf36f81e4f00ed7c67f159f2de11978470563f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Aug 2022 11:50:14 +0200 Subject: [PATCH 58/58] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ed0edc885..3ca2c9b2c 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.11.0 \ No newline at end of file +2.12.0 \ No newline at end of file