From 95f47c28fb8786f8d8d529192465bb6ec20db46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 3 Jun 2022 17:01:16 +0200 Subject: [PATCH 01/33] Make nix copy parallel again FILLME --- src/libstore/store-api.cc | 203 ++++++++++++++++++++------------------ src/libstore/store-api.hh | 7 ++ 2 files changed, 115 insertions(+), 95 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8861274a2..008451666 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -258,6 +258,86 @@ StorePath Store::addToStore( return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } +void Store::addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + std::atomic nrDone{0}; + std::atomic nrFailed{0}; + std::atomic bytesExpected{0}; + std::atomic nrRunning{0}; + + using PathWithInfo = std::pair>; + + std::map infosMap; + StorePathSet storePathsToAdd; + for (auto & thingToAdd : pathsToCopy) { + infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); + storePathsToAdd.insert(thingToAdd.first.path); + } + + auto showProgress = [&]() { + act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed); + }; + + ThreadPool pool; + + processGraph(pool, + storePathsToAdd, + + [&](const StorePath & path) { + auto & [info, source] = *infosMap.at(path); + /* auto storePathForDst = info.storePath; */ + /* if (info->ca && info->references.empty()) { */ + /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ + /* if (dstStore.storeDir == srcStore.storeDir) */ + /* assert(storePathForDst == storePath); */ + /* if (storePathForDst != storePath) */ + /* debug("replaced path '%s' to '%s' for substituter '%s'", */ + /* srcStore.printStorePath(storePath), */ + /* dstStore.printStorePath(storePathForDst), */ + /* dstStore.getUri()); */ + /* } */ + /* pathsMap.insert_or_assign(storePath, storePathForDst); */ + + if (isValidPath(info.path)) { + nrDone++; + showProgress(); + return StorePathSet(); + } + + bytesExpected += info.narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info.references; + }, + + [&](const StorePath & path) { + checkInterrupt(); + + auto & [info, source] = *infosMap.at(path); + + if (!isValidPath(info.path)) { + MaintainCount mc(nrRunning); + showProgress(); + try { + addToStore(info, *source, repair, checkSigs); + } catch (Error &e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what()); + showProgress(); + return; + } + } + + nrDone++; + showProgress(); + }); +} void Store::addMultipleToStore( Source & source, @@ -998,106 +1078,39 @@ std::map copyPaths( Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); - auto sorted = srcStore.topoSortPaths(missing); - std::reverse(sorted.begin(), sorted.end()); + /* auto sorted = srcStore.topoSortPaths(missing); */ + /* std::reverse(sorted.begin(), sorted.end()); */ - auto source = sinkToSource([&](Sink & sink) { - sink << sorted.size(); - for (auto & storePath : sorted) { - auto srcUri = srcStore.getUri(); - auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(storePath); - Activity act(*logger, lvlInfo, actCopyPath, - makeCopyPathMessage(srcUri, dstUri, storePathS), - {storePathS, srcUri, dstUri}); - PushActivity pact(act.id); + /* auto source = sinkToSource([&](Sink & sink) { */ + /* sink << sorted.size(); */ + /* for (auto & storePath : sorted) { */ + /* auto srcUri = srcStore.getUri(); */ + /* auto dstUri = dstStore.getUri(); */ + /* auto storePathS = srcStore.printStorePath(storePath); */ + /* Activity act(*logger, lvlInfo, actCopyPath, */ + /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ + /* {storePathS, srcUri, dstUri}); */ + /* PushActivity pact(act.id); */ - auto info = srcStore.queryPathInfo(storePath); - info->write(sink, srcStore, 16); - srcStore.narFromPath(storePath, sink); - } - }); + /* auto info = srcStore.queryPathInfo(storePath); */ + /* info->write(sink, srcStore, 16); */ + /* srcStore.narFromPath(storePath, sink); */ + /* } */ + /* }); */ - dstStore.addMultipleToStore(*source, repair, checkSigs); + std::vector>> pathsToCopy; + + for (auto & missingPath : missing) { + auto info = srcStore.queryPathInfo(missingPath); + auto source = sinkToSource([&](Sink & sink) { + srcStore.narFromPath(missingPath, sink); + }); + pathsToCopy.push_back(std::pair{*info, std::move(source)}); + } + + dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); #if 0 - std::atomic nrDone{0}; - std::atomic nrFailed{0}; - std::atomic bytesExpected{0}; - std::atomic nrRunning{0}; - - auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning, nrFailed); - }; - - ThreadPool pool; - - processGraph(pool, - StorePathSet(missing.begin(), missing.end()), - - [&](const StorePath & storePath) { - auto info = srcStore.queryPathInfo(storePath); - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (dstStore.isValidPath(storePath)) { - nrDone++; - showProgress(); - return StorePathSet(); - } - - bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); - - return info->references; - }, - - [&](const StorePath & storePath) { - checkInterrupt(); - - auto info = srcStore.queryPathInfo(storePath); - - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (!dstStore.isValidPath(storePathForDst)) { - MaintainCount mc(nrRunning); - showProgress(); - try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } catch (Error &e) { - nrFailed++; - if (!settings.keepGoing) - throw e; - printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); - showProgress(); - return; - } - } - - nrDone++; - showProgress(); - }); #endif return pathsMap; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0c8a4db56..d934979cf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "nar-info.hh" #include "realisation.hh" #include "path.hh" #include "derived-path.hh" @@ -364,6 +365,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs + ); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. From cb0553ecd0122f693653c1ac82beb26d127ce3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 14:03:46 +0200 Subject: [PATCH 02/33] Restore the "low-latency" ssh copying --- src/libstore/remote-store.cc | 17 +++++++++++++ src/libstore/remote-store.hh | 8 ++++++ src/libstore/store-api.cc | 49 ++++++++++++++++-------------------- src/libstore/store-api.hh | 6 ++++- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..ad2e5c18a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -673,6 +673,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } +void RemoteStore::addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto source = sinkToSource([&](Sink & sink) { + sink << pathsToCopy.size(); + for (auto & [pathInfo, pathSource] : pathsToCopy) { + pathInfo.write(sink, *this, 16); + pathSource->drainInto(sink); + } + }); + + addMultipleToStore(*source, repair, checkSigs); +} + void RemoteStore::addMultipleToStore( Source & source, RepairFlag repair, diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8493be6fc..5a599997e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -88,6 +88,14 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; + void addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; + + + StorePath addTextToStore( std::string_view name, std::string_view s, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 008451666..eeec6c6f7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -259,7 +259,7 @@ StorePath Store::addToStore( } void Store::addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) @@ -1072,37 +1072,33 @@ std::map copyPaths( for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + + // In the general case, `addMultipleToStore` requires a sorted list of + // store paths to add, so sort them right now + auto sortedMissing = srcStore.topoSortPaths(missing); + std::reverse(sortedMissing.begin(), sortedMissing.end()); + std::map pathsMap; for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + Store::PathsSource pathsToCopy; - /* auto sorted = srcStore.topoSortPaths(missing); */ - /* std::reverse(sorted.begin(), sorted.end()); */ - - /* auto source = sinkToSource([&](Sink & sink) { */ - /* sink << sorted.size(); */ - /* for (auto & storePath : sorted) { */ - /* auto srcUri = srcStore.getUri(); */ - /* auto dstUri = dstStore.getUri(); */ - /* auto storePathS = srcStore.printStorePath(storePath); */ - /* Activity act(*logger, lvlInfo, actCopyPath, */ - /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ - /* {storePathS, srcUri, dstUri}); */ - /* PushActivity pact(act.id); */ - - /* auto info = srcStore.queryPathInfo(storePath); */ - /* info->write(sink, srcStore, 16); */ - /* srcStore.narFromPath(storePath, sink); */ - /* } */ - /* }); */ - - std::vector>> pathsToCopy; - - for (auto & missingPath : missing) { + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); auto source = sinkToSource([&](Sink & sink) { + + // We can reasonably assume that the copy will happen whenever we + // read the path, so log something about that at that point + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(missingPath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + srcStore.narFromPath(missingPath, sink); }); pathsToCopy.push_back(std::pair{*info, std::move(source)}); @@ -1110,9 +1106,6 @@ std::map copyPaths( dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); - #if 0 - #endif - return pathsMap; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d934979cf..c0a61115b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,13 +360,17 @@ public: virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; + // A list of paths infos along with a source providing the content of the + // associated store path + using PathsSource = std::vector>>; + /* Import multiple paths into the store. */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); virtual void addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs From 480c2b6699e6afc6a746ba8a72319f197c3556ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 15:13:11 +0200 Subject: [PATCH 03/33] Rewrite the CA paths when moving them between store Bring back the possibility to copy CA paths with no reference (like the outputs of FO derivations or stuff imported at eval time) between stores that have a different prefix. --- src/libstore/store-api.cc | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index eeec6c6f7..61a12e84a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -289,18 +289,6 @@ void Store::addMultipleToStore( [&](const StorePath & path) { auto & [info, source] = *infosMap.at(path); - /* auto storePathForDst = info.storePath; */ - /* if (info->ca && info->references.empty()) { */ - /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ - /* if (dstStore.storeDir == srcStore.storeDir) */ - /* assert(storePathForDst == storePath); */ - /* if (storePathForDst != storePath) */ - /* debug("replaced path '%s' to '%s' for substituter '%s'", */ - /* srcStore.printStorePath(storePath), */ - /* dstStore.printStorePath(storePathForDst), */ - /* dstStore.getUri()); */ - /* } */ - /* pathsMap.insert_or_assign(storePath, storePathForDst); */ if (isValidPath(info.path)) { nrDone++; @@ -1085,10 +1073,32 @@ std::map copyPaths( Store::PathsSource pathsToCopy; + auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath { + auto storePathForSrc = currentPathInfo.path; + auto storePathForDst = storePathForSrc; + if (currentPathInfo.ca && currentPathInfo.references.empty()) { + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); + if (dstStore.storeDir == srcStore.storeDir) + assert(storePathForDst == storePathForSrc); + if (storePathForDst != storePathForSrc) + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePathForSrc), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); + } + return storePathForDst; + }; + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); - auto source = sinkToSource([&](Sink & sink) { + auto storePathForDst = computeStorePathForDst(*info); + pathsMap.insert_or_assign(missingPath, storePathForDst); + + ValidPathInfo infoForDst = *info; + infoForDst.path = storePathForDst; + + auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we // read the path, so log something about that at that point auto srcUri = srcStore.getUri(); @@ -1101,7 +1111,7 @@ std::map copyPaths( srcStore.narFromPath(missingPath, sink); }); - pathsToCopy.push_back(std::pair{*info, std::move(source)}); + pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); From 34d90fbe224e7b626366f98ea1bc9e370d9cb534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 15:25:52 +0200 Subject: [PATCH 04/33] Mention the parallel copy in the release notes --- doc/manual/src/release-notes/rl-next.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 52e3b6240..6c2c4689c 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,3 +2,7 @@ * Nix can now be built with LTO by passing `--enable-lto` to `configure`. LTO is currently only supported when building with GCC. + +* `nix copy` now copies the store paths in parallel as much as possible (again). + This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything + in one batch to avoid latencies issues. From 56f6f3725f4fbeeb7900ae95bd71d559695b3dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 19 Jul 2022 19:46:00 +0200 Subject: [PATCH 05/33] Don't ultimately trust the signed paths Like the old implem did (and like you'd want it to be anyways) --- src/libstore/store-api.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 61a12e84a..06d03bff2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -305,7 +305,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info, source] = *infosMap.at(path); + auto & [info_, source] = *infosMap.at(path); + auto info = info_; + info.ultimate = false; if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); From 5f37c5191a3a8f5c7ab31a0dd8bffe14aaa6b76c Mon Sep 17 00:00:00 2001 From: Winter Date: Tue, 9 Aug 2022 16:48:34 -0400 Subject: [PATCH 06/33] nix-shell: specify which outputs from bashInteractive to build --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 7eb8c8f6a..df292dce6 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -401,7 +401,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { .drvPath = bashDrv, - .outputs = {}, + .outputs = {"out"}, }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; From 0eb9946e1d3621cfc2fcffc9378dba334b25fb26 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 9 Aug 2022 23:21:09 -0400 Subject: [PATCH 07/33] docker.nix: Provide boolean for whether to bundle nixpkgs --- docker.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index 8e6aa227f..e95caf274 100644 --- a/docker.nix +++ b/docker.nix @@ -2,6 +2,7 @@ , lib ? pkgs.lib , name ? "nix" , tag ? "latest" +, bundleNixpkgs ? true , channelName ? "nixpkgs" , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" , extraPkgs ? [] @@ -139,10 +140,12 @@ let baseSystem = let nixpkgs = pkgs.path; - channel = pkgs.runCommand "channel-nixos" { } '' + channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } '' mkdir $out - ln -s ${nixpkgs} $out/nixpkgs - echo "[]" > $out/manifest.nix + if [ "$bundleNixpkgs" ]; then + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + fi ''; rootEnv = pkgs.buildPackages.buildEnv { name = "root-profile-env"; From 703b335c1d187fb18ca5a5450b33e4e9a98abc7c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 15 Aug 2022 11:34:39 +0200 Subject: [PATCH 08/33] reword description of language properties the list style is supposed to give more structure. each property is explained as concisely as possible while trying not to sound too fancy. --- doc/manual/src/language/index.md | 38 ++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index c4b3abf75..a4b402f8b 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,13 +1,33 @@ # Nix Language -The Nix language is a pure, lazy, functional language. Purity -means that operations in the language don't have side-effects (for -instance, there is no variable assignment). Laziness means that -arguments to functions are evaluated only when they are needed. -Functional means that functions are “normal” values that can be passed -around and manipulated in interesting ways. The language is not a -full-featured, general purpose language. Its main job is to describe -packages, compositions of packages, and the variability within packages. +The Nix language is -This section presents the various features of the language. +- *domain-specific* + + It only exists for the Nix package manager: + to describe packages and configurations as well as their variants and compositions. + It is not intended for general purpose use. + +- *declarative* + + There is no notion of executing sequential steps. + Dependencies between operations are established only through data. + +- *pure* + + Values cannot change during computation. + Functions always produce the same output if their input does not change. + +- *functional* + + Functions are like any other value. + Functions can be assigned to names, taken as arguments, or returned by functions. + +- *lazy* + + Expressions are only evaluated when their value is needed. + +- *dynamically typed* + + Type errors are only detected when expressions are evaluated. From 6547dcde2a37179f98cc9a8702bc15a2fd1e6a4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:41:19 +0200 Subject: [PATCH 09/33] Use plain mktemp This fixes the case where $TMPDIR doesn't end in a slash. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a9f3e74dc..7c8e159b5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,7 +349,7 @@ _sudo() { } -readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" } From 823e1017d809219e7e00b24ef4ccb0a8b568449c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:47:01 +0200 Subject: [PATCH 10/33] Ensure that $TMPDIR exists if defined --- scripts/install-multi-user.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7c8e159b5..1431857d5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,6 +349,11 @@ _sudo() { } +# Ensure that $TMPDIR exists if defined. +if [[ -v TMPDIR ]]; then + mkdir -m 0700 -p "$TMPDIR" +fi + readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" From 8188b1d0abc2eba6497b5dc47f7e848cbacb7677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:59:04 +0200 Subject: [PATCH 11/33] json: write null on abnormal placeholder destruction Avoids leaving dangling attributes like { "foo": } in case of exceptions. --- src/libutil/json.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libutil/json.cc b/src/libutil/json.cc index abe0e6e74..2f9e97ff5 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -193,7 +193,11 @@ JSONObject JSONPlaceholder::object() JSONPlaceholder::~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + if (first) { + assert(std::uncaught_exceptions()); + if (state->stack != 0) + write(nullptr); + } } } From 7535ee345da6c0aea1dd81e7de7725b37d8bf8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:33:46 +0200 Subject: [PATCH 12/33] nix-env: don't output incomplete JSON --- src/nix-env/nix-env.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..fdd66220a 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -940,12 +940,12 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin JSONObject metaObj = pkgObj.object("meta"); StringSet metaNames = i.queryMetaNames(); for (auto & j : metaNames) { - auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - placeholder.write(nullptr); + metaObj.attr(j, nullptr); } else { + auto placeholder = metaObj.placeholder(j); PathSet context; printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } From 0d2bf7acf994ba331d6f72c746721b354931be76 Mon Sep 17 00:00:00 2001 From: Solene Rapenne Date: Fri, 19 Aug 2022 12:40:22 +0200 Subject: [PATCH 13/33] add a nix.conf option to set a download speed limit --- src/libstore/filetransfer.cc | 3 +++ src/libstore/globals.hh | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8454ad7d2..252403cb5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + if (settings.downloadSpeed.get() > 0) + curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..ca8fc6d5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -746,6 +746,13 @@ public: /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. )"}; + Setting downloadSpeed { + this, 0, "download-speed", + R"( + Specify the maxium transfer rate in kilobytes per second you want + nix to use for download. + )"}; + Setting netrcFile{ this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", R"( From 0bf52b73f4cb61bc12c95a015a7be45f7174ca01 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 19 Aug 2022 15:03:37 -0500 Subject: [PATCH 14/33] install: only create TMPDIR if missing --- scripts/install-multi-user.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 1431857d5..01dbf0c0e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -348,10 +348,9 @@ _sudo() { fi } - # Ensure that $TMPDIR exists if defined. -if [[ -v TMPDIR ]]; then - mkdir -m 0700 -p "$TMPDIR" +if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then + mkdir -m 0700 -p "${TMPDIR:-}" fi readonly SCRATCH=$(mktemp -d) From 7d800909e94c482a2093bc95a2f3dca565c148b2 Mon Sep 17 00:00:00 2001 From: Jakub Kuczys Date: Sat, 20 Aug 2022 03:48:42 +0200 Subject: [PATCH 15/33] Fix default profile path for root in nix profile documentation --- src/nix/profile.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.md b/src/nix/profile.md index 8dade051d..be3c5ba1a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. The default profile used by `nix profile` is `$HOME/.nix-profile`, which, if it does not exist, is created as a symlink to -`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`/nix/var/nix/profiles/default` if Nix is invoked by the `root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. You can specify another profile location using `--profile` *path*. From caad87e6dbfbc62fafb4a055b45d7f2eb3d11efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Sat, 20 Aug 2022 18:21:36 +0200 Subject: [PATCH 16/33] Better documentation wording Co-authored-by: Anderson Torres --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca8fc6d5f..1ff7d2d0a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -749,7 +749,7 @@ public: Setting downloadSpeed { this, 0, "download-speed", R"( - Specify the maxium transfer rate in kilobytes per second you want + Specify the maximum transfer rate in kilobytes per second you want nix to use for download. )"}; From c21b1a7e67cc28b6e95a563daa786f385bc716b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:14:14 +0200 Subject: [PATCH 17/33] Spelling --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1ff7d2d0a..e9d721e59 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -750,7 +750,7 @@ public: this, 0, "download-speed", R"( Specify the maximum transfer rate in kilobytes per second you want - nix to use for download. + Nix to use for downloads. )"}; Setting netrcFile{ From 0d2163c6dcf03463fa91ec6d0d96c928ad907366 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:27:36 +0200 Subject: [PATCH 18/33] nix repl: Stop the progress bar The repl was broken since c3769c68465bae971ab6bb48cfcdea85b61ea83a. In general, the progress bar is incompatible with the repl. --- src/libcmd/repl.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 23df40337..150bd42ac 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -35,6 +35,7 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" +#include "progress-bar.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -252,6 +253,10 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif + /* Stop the progress bar because it interferes with the display of + the repl. */ + stopProgressBar(); + std::string input; while (true) { @@ -1037,9 +1042,10 @@ void runRepl( struct CmdRepl : InstallablesCommand { - CmdRepl(){ + CmdRepl() { evalSettings.pureEval = false; } + void prepare() { if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { @@ -1053,12 +1059,15 @@ struct CmdRepl : InstallablesCommand } installables = InstallablesCommand::load(); } + std::vector files; + Strings getDefaultFlakeAttrPaths() override { return {""}; } - virtual bool useDefaultInstallables() override + + bool useDefaultInstallables() override { return file.has_value() or expr.has_value(); } From 062e4fcdde145ec6780df8d1002dc7380f6eb4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 16 Aug 2022 12:23:37 +0200 Subject: [PATCH 19/33] JSON: print paths as strings without copying them to the store Makes `printValueAsJSON` not copy paths to the store for `nix eval --json`, `nix-instantiate --eval --json` and `nix-env --json`. Fixes https://github.com/NixOS/nix/issues/5612 --- src/libexpr/value-to-json.cc | 21 ++++++++++++--------- src/libexpr/value-to-json.hh | 4 ++-- src/libexpr/value.hh | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/eval.cc | 2 +- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 03504db61..4d63d8b49 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,7 +10,7 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context) + Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore) { checkInterrupt(); @@ -32,7 +32,10 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nPath: - out.write(state.copyPathToStore(context, v.path)); + if (copyToStore) + out.write(state.copyPathToStore(context, v.path)); + else + out.write(v.path); break; case nNull: @@ -54,10 +57,10 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context); + printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore); } } else - printValueAsJSON(state, strict, *i->value, i->pos, out, context); + printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore); break; } @@ -65,13 +68,13 @@ void printValueAsJSON(EvalState & state, bool strict, auto list(out.list()); for (auto elem : v.listItems()) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *elem, pos, placeholder, context); + printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore); } break; } case nExternal: - v.external->printValueAsJSON(state, strict, out, context); + v.external->printValueAsJSON(state, strict, out, context, copyToStore); break; case nFloat: @@ -91,14 +94,14 @@ void printValueAsJSON(EvalState & state, bool strict, } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context) + Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, pos, out, context); + printValueAsJSON(state, strict, v, pos, out, context, copyToStore); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const + JSONPlaceholder & out, PathSet & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index c020a817a..7ddc8a5b1 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context); + Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context); + Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 2008df74d..590ba7783 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -99,7 +99,7 @@ class ExternalValueBase /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual void printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const; + JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const; /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..c4e981d49 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -947,7 +947,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context, false); } } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index d3144e131..6181d2190 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -53,7 +53,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); else { if (strict) state.forceValueDeep(vRes); vRes.print(state.symbols, std::cout); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 967dc8519..cc022ae43 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -116,7 +116,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, pos, jsonOut, context); + printValueAsJSON(*state, true, *v, pos, jsonOut, context, false); } else { From 4c2ff4a0f4c2106d5792a87a1ba9ee1fd18c0e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 22 Aug 2022 15:07:52 +0200 Subject: [PATCH 20/33] JSON: add missing newlines after `nix eval --json` and `nix-instantiate --eval --json`. --- src/nix-instantiate/nix-instantiate.cc | 5 +++-- src/nix/eval.cc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6181d2190..80f35828c 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -52,9 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths, state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); - else if (output == okJSON) + else if (output == okJSON) { printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); - else { + std::cout << std::endl; + } else { if (strict) state.forceValueDeep(vRes); vRes.print(state.symbols, std::cout); std::cout << std::endl; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index cc022ae43..ddd2790c6 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -117,6 +117,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, pos, jsonOut, context, false); + std::cout << std::endl; } else { From f865048332a26f69a007881877203b9428783357 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 15:30:38 +0200 Subject: [PATCH 21/33] Indentation --- src/libstore/remote-store.hh | 10 ++++------ src/libstore/store-api.hh | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 5a599997e..11d089cd2 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -89,12 +89,10 @@ public: CheckSigsFlag checkSigs) override; void addMultipleToStore( - PathsSource & pathsToCopy, - Activity & act, - RepairFlag repair, - CheckSigsFlag checkSigs) override; - - + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c0a61115b..c8a667c6d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -369,12 +369,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, - CheckSigsFlag checkSigs = CheckSigs - ); + CheckSigsFlag checkSigs = CheckSigs); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. From f0358ed4650e4608a383bd9f59ee23545f86c4ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Aug 2022 14:14:47 +0200 Subject: [PATCH 22/33] Fix a hang in nix-copy-ssh.sh This hang for some reason didn't trigger in the Nix build, but did running 'make installcheck' interactively. What happened: * Store::addMultipleToStore() calls a SinkToSource object to copy a path, which in turn calls LegacySSHStore::narFromPath(), which acquires a connection. * The SinkToSource object is not destroyed after the last bytes has been read, so the coroutine's stack is still alive and its destructors are not run. So the connection is not released. * Then when the next path is copied, because max-connections = 1, LegacySSHStore::narFromPath() hangs forever waiting for a connection to be released. The fix is to make sure that the source object is destroyed when we're done with it. --- src/libstore/store-api.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1406bf657..9c3a0b3d6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,6 +11,7 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include "finally.hh" #include @@ -271,7 +272,7 @@ void Store::addMultipleToStore( using PathWithInfo = std::pair>; - std::map infosMap; + std::map infosMap; StorePathSet storePathsToAdd; for (auto & thingToAdd : pathsToCopy) { infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); @@ -288,7 +289,8 @@ void Store::addMultipleToStore( storePathsToAdd, [&](const StorePath & path) { - auto & [info, source] = *infosMap.at(path); + + auto & [info, _] = *infosMap.at(path); if (isValidPath(info.path)) { nrDone++; @@ -309,12 +311,21 @@ void Store::addMultipleToStore( auto info = info_; info.ultimate = false; + /* Make sure that the Source object is destroyed when + we're done. In particular, a SinkToSource object must + be destroyed to ensure that the destructors on its + stack frame are run; this includes + LegacySSHStore::narFromPath()'s connection lock. */ + Finally cleanupSource{[&]() { + source.reset(); + }}; + if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); showProgress(); try { addToStore(info, *source, repair, checkSigs); - } catch (Error &e) { + } catch (Error & e) { nrFailed++; if (!settings.keepGoing) throw e; From ff0b5a778c41a94075d7c651477c4a3a8b4b00cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 23 Aug 2022 14:40:27 +0200 Subject: [PATCH 23/33] Revert to `copyStore = true` for `nix-instantiate` and `nix-env` --- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c4e981d49..a69d3700d 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -947,7 +947,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context, false); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 80f35828c..6b5ba595d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -53,7 +53,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) { - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); std::cout << std::endl; } else { if (strict) state.forceValueDeep(vRes); From db026103b18fb8b5a719594502edd0f89eb9c268 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Aug 2022 14:57:08 +0200 Subject: [PATCH 24/33] nix develop: Ignore some more bash special variables Fixes #6940. --- src/nix/get-env.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 42c806450..a7a8a01b9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -43,6 +43,7 @@ __dumpEnv() { local __var_name="${BASH_REMATCH[2]}" if [[ $__var_name =~ ^BASH_ || \ + $__var_name =~ ^COMP_ || \ $__var_name = _ || \ $__var_name = DIRSTACK || \ $__var_name = EUID || \ @@ -54,7 +55,9 @@ __dumpEnv() { $__var_name = PWD || \ $__var_name = RANDOM || \ $__var_name = SHLVL || \ - $__var_name = SECONDS \ + $__var_name = SECONDS || \ + $__var_name = EPOCHREALTIME || \ + $__var_name = EPOCHSECONDS \ ]]; then continue; fi if [[ -z $__first ]]; then printf ',\n'; else __first=; fi From 4d4f2d10e7ffceac7a700abe0004ffb5b2b23b33 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Tue, 23 Aug 2022 19:38:53 -0400 Subject: [PATCH 25/33] darwin-install: fix shell hint --- scripts/install-darwin-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index afaa6783b..5111a5dde 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -167,7 +167,7 @@ poly_user_shell_get() { } poly_user_shell_set() { - _sudo "in order to give $1 a safe home directory" \ + _sudo "in order to give $1 a safe shell" \ /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" } From c2d74569269d9642448a701aa6efe08ec0fe6f00 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 24 Aug 2022 01:54:43 +0200 Subject: [PATCH 26/33] Fix a misplaced parenthese in serve protocol check This issue made it impossible for clients using a serve protocol of version <= 2.3 to use the `cmdBuildDerivation` command of servers using a protocol of version >= 2.6. The faulty version check makes the server send back build outputs that the client is not expecting. --- src/nix-store/nix-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b453ea1ca..23f2ad3cf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -922,7 +922,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion >= 6)) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { worker_proto::write(*store, out, status.builtOutputs); } From 8d906b1f3bd4343b6b309ddfca824d5dd00a09b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:11:03 +0200 Subject: [PATCH 27/33] Fix macOS build --- src/libstore/store-api.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9c3a0b3d6..2cd6c15ec 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -307,8 +307,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info_, source] = *infosMap.at(path); + auto & [info_, source_] = *infosMap.at(path); auto info = info_; + auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when From 56d97d4b4df02e3464a2f003a90b7f6abae16722 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:49:58 +0200 Subject: [PATCH 28/33] Remove redundant Finally --- src/libstore/store-api.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2cd6c15ec..86b12257a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,7 +11,6 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" -#include "finally.hh" #include @@ -309,7 +308,6 @@ void Store::addMultipleToStore( auto & [info_, source_] = *infosMap.at(path); auto info = info_; - auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when @@ -317,9 +315,7 @@ void Store::addMultipleToStore( be destroyed to ensure that the destructors on its stack frame are run; this includes LegacySSHStore::narFromPath()'s connection lock. */ - Finally cleanupSource{[&]() { - source.reset(); - }}; + auto source = std::move(source_); if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); From a17ce0a8a9c3dd96d0a5043a80acab0c6810a199 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 21:19:43 +0200 Subject: [PATCH 29/33] Fix evaluation cache 98e361ad4c1a26d4ffe4762a6f33bb9e39321a39 introduced a regression where previously stored attributes were replaced by placeholders. As a result, a command like 'nix build nixpkgs#hello' had to be executed at least twice to get caching. This code does not seem necessary for suggestions to work. --- src/libexpr/eval-cache.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0d83b6cfe..b259eec63 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -507,11 +507,6 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - for (auto & attr : *v.attrs) { - if (root->db) - root->db->setPlaceholder({cachedValue->first, attr.name}); - } - auto attr = v.attrs->get(name); if (!attr) { From bb411e4ae16d6a5c61ea595c0c12e2ecee081ff9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:36:40 +0200 Subject: [PATCH 30/33] Fix progress bar flicker with -L This was caused by -L calling setLogFormat() again, which caused the creation of a new progress bar without destroying the old one. So we had two progress bars clobbering each other. We should change 'logger' to be a smart pointer, but I'll do that in a future PR. Fixes #6931. --- src/libmain/loggers.cc | 7 +++++-- src/libmain/progress-bar.cc | 23 ++++++++++++----------- src/libmain/progress-bar.hh | 4 ++-- src/libutil/logging.hh | 3 +++ src/nix/main.cc | 2 +- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cdf23859b..cda5cb939 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -30,8 +30,11 @@ Logger * makeDefaultLogger() { return makeJSONLogger(*makeSimpleLogger(true)); case LogFormat::bar: return makeProgressBar(); - case LogFormat::barWithLogs: - return makeProgressBar(true); + case LogFormat::barWithLogs: { + auto logger = makeProgressBar(); + logger->setPrintBuildLogs(true); + return logger; + } default: abort(); } diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 5183f212f..0bbeaff8d 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -81,14 +81,13 @@ private: std::condition_variable quitCV, updateCV; - bool printBuildLogs; + bool printBuildLogs = false; bool isTTY; public: - ProgressBar(bool printBuildLogs, bool isTTY) - : printBuildLogs(printBuildLogs) - , isTTY(isTTY) + ProgressBar(bool isTTY) + : isTTY(isTTY) { state_.lock()->active = isTTY; updateThread = std::thread([&]() { @@ -503,19 +502,21 @@ public: draw(*state); return s[0]; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { + this->printBuildLogs = printBuildLogs; + } }; -Logger * makeProgressBar(bool printBuildLogs) +Logger * makeProgressBar() { - return new ProgressBar( - printBuildLogs, - shouldANSI() - ); + return new ProgressBar(shouldANSI()); } -void startProgressBar(bool printBuildLogs) +void startProgressBar() { - logger = makeProgressBar(printBuildLogs); + logger = makeProgressBar(); } void stopProgressBar() diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 7f0dafecf..3a76f8448 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -4,9 +4,9 @@ namespace nix { -Logger * makeProgressBar(bool printBuildLogs = false); +Logger * makeProgressBar(); -void startProgressBar(bool printBuildLogs = false); +void startProgressBar(); void stopProgressBar(); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 6f81b92de..d0817b4a9 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -111,6 +111,9 @@ public: virtual std::optional ask(std::string_view s) { return {}; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { } }; ActivityId getCurActivity(); diff --git a/src/nix/main.cc b/src/nix/main.cc index a8404a2ea..f434e9655 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -82,7 +82,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .shortName = 'L', .description = "Print full build logs on standard error.", .category = loggingCategory, - .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, + .handler = {[&]() { logger->setPrintBuildLogs(true); }}, }); addFlag({ From d046eb1463fad967d47cc5becfe5b7a08b5adfa8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:42:34 +0200 Subject: [PATCH 31/33] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index f161b5d80..ed0edc885 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.10.0 \ No newline at end of file +2.11.0 \ No newline at end of file From b0488a29dc7401e5ecd9221215da5ea9879e56d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:44:58 +0200 Subject: [PATCH 32/33] Branch 2.11 release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.11.md | 5 +++++ doc/manual/src/release-notes/rl-next.md | 3 --- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.11.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8fbb59716..9b66ec3db 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -73,6 +73,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) diff --git a/doc/manual/src/release-notes/rl-2.11.md b/doc/manual/src/release-notes/rl-2.11.md new file mode 100644 index 000000000..b322a4e5e --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.11.md @@ -0,0 +1,5 @@ +# Release 2.11 (2022-08-24) + +* `nix copy` now copies the store paths in parallel as much as possible (again). + This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything + in one batch to avoid latencies issues. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7d82c3dc4..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,2 @@ # Release X.Y (202?-??-??) -* `nix copy` now copies the store paths in parallel as much as possible (again). - This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything - in one batch to avoid latencies issues. \ No newline at end of file From 57cf36f81e4f00ed7c67f159f2de11978470563f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Aug 2022 11:50:14 +0200 Subject: [PATCH 33/33] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ed0edc885..3ca2c9b2c 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.11.0 \ No newline at end of file +2.12.0 \ No newline at end of file