From 452ffe5464f20ac44a01c536349895d138150a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 18 May 2022 16:46:13 +0200 Subject: [PATCH 001/119] Hint at the source file on conflict in `flake new` Add a pointer to the source file (from the template) when `nix flake new` (or `init`) encounters an already existing file Fix #6542 --- src/nix/flake.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1938ce4e6..a1edb5dbf 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -758,7 +758,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand if (pathExists(to2)) { auto contents2 = readFile(to2); if (contents != contents2) - throw Error("refusing to overwrite existing file '%s'", to2); + throw Error("refusing to overwrite existing file '%s' - please merge manually with '%s'", to2, from2); } else writeFile(to2, contents); } @@ -766,7 +766,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto target = readLink(from2); if (pathExists(to2)) { if (readLink(to2) != target) - throw Error("refusing to overwrite existing symlink '%s'", to2); + throw Error("refusing to overwrite existing symlink '%s' - please merge manually with '%s'", to2, from2); } else createSymlink(target, to2); } From 06d57ce7597fc1b49ce1cdc721edc64eaafe38fb Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Wed, 22 Dec 2021 15:36:08 -0700 Subject: [PATCH 002/119] nix repl: load flakes from cli args If experimental feature "flakes" is enabled, args passed to `nix repl` will now be considered flake refs and imported using the existing `:load-flake` machinery. In addition, `:load-flake` now supports loading flake fragments. --- src/nix/repl.cc | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 2967632ed..d20eb0929 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -646,11 +646,11 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); - Value v; + auto v = state->allocValue(); flake::callFlake(*state, flake::lockFlake(*state, flakeRef, @@ -659,8 +659,17 @@ void NixRepl::loadFlake(const std::string & flakeRefS) .useRegistries = !evalSettings.pureEval, .allowMutable = !evalSettings.pureEval, }), - v); - addAttrsToScope(v); + *v); + + auto f = v->attrs->get(state->symbols.create(fragment)); + + if (f == 0) { + warn("no attribute %s, nothing loaded", fragment); + return; + }; + + fragment != "" ? addAttrsToScope(*f->value) : addAttrsToScope(*v); + } @@ -689,7 +698,10 @@ void NixRepl::reloadFiles() if (!first) notice(""); first = false; notice("Loading '%1%'...", i); - loadFile(i); + + settings.isExperimentalFeatureEnabled(Xp::Flakes) + ? loadFlake(i) + : loadFile(i); } } From 81567a096258026148b42f3048be9b2ba295b41a Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 18 Feb 2022 18:33:03 -0500 Subject: [PATCH 003/119] repl: allow loading installables from CLI repl: search installable with findAlongAttrPath repl: refactor handling of args repl: temp --- src/libcmd/command.hh | 1 + src/libcmd/installables.cc | 5 ++- src/nix/repl.cc | 87 ++++++++++++++++++++++---------------- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 078e2a2ce..65626e33f 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -132,6 +132,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand InstallableCommand(bool supportReadOnlyMode = false); void prepare() override; + std::shared_ptr load(); std::optional getFlakeRefForCompletion() override { diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 635ce19b6..7d2ff0f68 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1054,10 +1054,13 @@ InstallableCommand::InstallableCommand(bool supportReadOnlyMode) }} }); } +std::shared_ptr InstallableCommand::load() { + return parseInstallable(getStore(), _installable); +} void InstallableCommand::prepare() { - installable = parseInstallable(getStore(), _installable); + installable = load(); } } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d20eb0929..df921ef06 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -22,6 +22,7 @@ extern "C" { #include "ansicolor.hh" #include "shared.hh" #include "eval.hh" +#include "eval-cache.hh" #include "eval-inline.hh" #include "attr-path.hh" #include "store-api.hh" @@ -42,16 +43,20 @@ extern "C" { namespace nix { +typedef std::vector> Installables; + struct NixRepl #if HAVE_BOEHMGC : gc #endif { std::string curDir; - std::unique_ptr state; + ref state; Bindings * autoArgs; Strings loadedFiles; + typedef std::vector> AnnotatedValues; + std::function getValues; const static int envSize = 32768; StaticEnv staticEnv; @@ -61,13 +66,16 @@ struct NixRepl const Path historyFile; - NixRepl(const Strings & searchPath, nix::ref store); + NixRepl(const Strings & searchPath, nix::ref store,ref state, + std::function getValues); ~NixRepl(); - void mainLoop(const std::vector & files); + void mainLoop(); StringSet completePrefix(const std::string & prefix); bool getLine(std::string & input, const std::string &prompt); StorePath getDerivationPath(Value & v); bool processLine(std::string line); + + void loadInstallable(Installable & installable); void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); void initEnv(); @@ -92,8 +100,10 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store) - : state(std::make_unique(searchPath, store)) +NixRepl::NixRepl(const Strings & searchPath, nix::ref store,ref state, + std::function getValues) + : state(state) + , getValues(getValues) , staticEnv(false, &state->staticBaseEnv) , historyFile(getDataDir() + "/nix/repl-history") { @@ -198,16 +208,12 @@ namespace { } } -void NixRepl::mainLoop(const std::vector & files) +void NixRepl::mainLoop() { std::string error = ANSI_RED "error:" ANSI_NORMAL " "; notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n"); - for (auto & i : files) - loadedFiles.push_back(i); - reloadFiles(); - if (!loadedFiles.empty()) notice(""); // Allow nix-repl specific settings in .inputrc rl_readline_name = "nix-repl"; @@ -630,6 +636,11 @@ bool NixRepl::processLine(std::string line) return true; } +void NixRepl::loadInstallable(Installable & installable) +{ + auto [val, pos] = installable.toValue(*state); + addAttrsToScope(*val); +} void NixRepl::loadFile(const Path & path) { @@ -646,11 +657,11 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); - auto v = state->allocValue(); + Value v; flake::callFlake(*state, flake::lockFlake(*state, flakeRef, @@ -659,17 +670,8 @@ void NixRepl::loadFlake(const std::string & flakeRefS) .useRegistries = !evalSettings.pureEval, .allowMutable = !evalSettings.pureEval, }), - *v); - - auto f = v->attrs->get(state->symbols.create(fragment)); - - if (f == 0) { - warn("no attribute %s, nothing loaded", fragment); - return; - }; - - fragment != "" ? addAttrsToScope(*f->value) : addAttrsToScope(*v); - + v); + addAttrsToScope(v); } @@ -693,15 +695,14 @@ void NixRepl::reloadFiles() Strings old = loadedFiles; loadedFiles.clear(); - bool first = true; for (auto & i : old) { - if (!first) notice(""); - first = false; notice("Loading '%1%'...", i); + loadFile(i); + } - settings.isExperimentalFeatureEnabled(Xp::Flakes) - ? loadFlake(i) - : loadFile(i); + for (auto & [i,what] : getValues()) { + notice("Loading Installable '%1%'...", what); + addAttrsToScope(*i); } } @@ -898,17 +899,20 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m return str; } -struct CmdRepl : StoreCommand, MixEvalArgs +struct CmdRepl : InstallableCommand { std::vector files; + Strings getDefaultFlakeAttrPathPrefixes() + override { + return {}; + } + Strings getDefaultFlakeAttrPaths() + override { + return {""}; + } CmdRepl() { - expectArgs({ - .label = "files", - .handler = {&files}, - .completer = completePath - }); } std::string description() override @@ -925,10 +929,19 @@ struct CmdRepl : StoreCommand, MixEvalArgs void run(ref store) override { + evalSettings.pureEval = false; - auto repl = std::make_unique(searchPath, openStore()); + auto state = getEvalState(); + auto repl = std::make_unique(searchPath, openStore(),state + ,[&]()->NixRepl::AnnotatedValues{ + auto installable = load(); + auto [val, pos] = installable->toValue(*state); + auto what = installable->what(); + return { {val,what} }; + } + ); repl->autoArgs = getAutoArgs(*repl->state); - repl->mainLoop(files); + repl->mainLoop(); } }; From 5640b528349c43717aa501797a4f337373ebf3e4 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 11 Mar 2022 13:26:08 -0500 Subject: [PATCH 004/119] repl: use installables --- src/libcmd/command.hh | 2 +- src/libcmd/installables.cc | 12 +++++++----- src/libcmd/installables.hh | 1 + src/nix/repl.cc | 23 ++++++++++++----------- src/nix/repl.md | 30 +++++++++++++++++++++++++++--- 5 files changed, 48 insertions(+), 20 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 65626e33f..65eb0c4a0 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -114,6 +114,7 @@ struct InstallablesCommand : virtual Args, SourceExprCommand InstallablesCommand(); void prepare() override; + Installables load(); virtual bool useDefaultInstallables() { return true; } @@ -132,7 +133,6 @@ struct InstallableCommand : virtual Args, SourceExprCommand InstallableCommand(bool supportReadOnlyMode = false); void prepare() override; - std::shared_ptr load(); std::optional getFlakeRefForCompletion() override { diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 7d2ff0f68..c29fbeec9 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1025,11 +1025,16 @@ InstallablesCommand::InstallablesCommand() void InstallablesCommand::prepare() { + installables = load(); +} + +Installables InstallablesCommand::load() { + Installables installables; if (_installables.empty() && useDefaultInstallables()) // FIXME: commands like "nix profile install" should not have a // default, probably. _installables.push_back("."); - installables = parseInstallables(getStore(), _installables); + return parseInstallables(getStore(), _installables); } std::optional InstallablesCommand::getFlakeRefForCompletion() @@ -1054,13 +1059,10 @@ InstallableCommand::InstallableCommand(bool supportReadOnlyMode) }} }); } -std::shared_ptr InstallableCommand::load() { - return parseInstallable(getStore(), _installable); -} void InstallableCommand::prepare() { - installable = load(); + installable = parseInstallable(getStore(), _installable); } } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 5d715210e..b97888db6 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -131,6 +131,7 @@ struct Installable OperateOn operateOn, const std::vector> & installables); }; +typedef std::vector> Installables; struct InstallableValue : Installable { diff --git a/src/nix/repl.cc b/src/nix/repl.cc index df921ef06..b5ecc8ad0 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -43,8 +43,6 @@ extern "C" { namespace nix { -typedef std::vector> Installables; - struct NixRepl #if HAVE_BOEHMGC : gc @@ -899,17 +897,16 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m return str; } -struct CmdRepl : InstallableCommand +struct CmdRepl : InstallablesCommand { std::vector files; - Strings getDefaultFlakeAttrPathPrefixes() - override { - return {}; - } Strings getDefaultFlakeAttrPaths() override { return {""}; } + virtual bool useDefaultInstallables() { + return file.has_value() or expr.has_value(); + } CmdRepl() { @@ -934,10 +931,14 @@ struct CmdRepl : InstallableCommand auto state = getEvalState(); auto repl = std::make_unique(searchPath, openStore(),state ,[&]()->NixRepl::AnnotatedValues{ - auto installable = load(); - auto [val, pos] = installable->toValue(*state); - auto what = installable->what(); - return { {val,what} }; + auto installables = load(); + NixRepl::AnnotatedValues values; + for (auto & installable: installables){ + auto [val, pos] = installable->toValue(*state); + auto what = installable->what(); + values.push_back( {val,what} ); + } + return values; } ); repl->autoArgs = getAutoArgs(*repl->state); diff --git a/src/nix/repl.md b/src/nix/repl.md index 9b6f2bee3..be1498e5b 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -24,10 +24,34 @@ R""( * Interact with Nixpkgs in the REPL: ```console - # nix repl '' + # nix repl --file example.nix + Loading Installable ''... + Added 3 variables. - Loading ''... - Added 12428 variables. + # nix repl --expr '{a={b=3;c=4;};}' + Loading Installable ''... + Added 1 variables. + + # nix repl --expr '{a={b=3;c=4;};}' a + Loading Installable ''... + Added 1 variables. + + # nix repl nixpkgs + Loading Installable 'flake:nixpkgs#'... + Added 5 variables. + + nix-repl> legacyPackages.x86_64-linux.emacs.name + "emacs-27.1" + + nix-repl> legacyPackages.x86_64-linux.emacs.name + "emacs-27.1" + + nix-repl> :q + + # nix repl --expr 'import {}' --impure + + Loading Installable ''... + Added 12439 variables. nix-repl> emacs.name "emacs-27.1" From 1ca3f6035da4e82647382405c774e43e02de3fa1 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 11 Mar 2022 13:52:08 -0500 Subject: [PATCH 005/119] repl: update docs with installables --- doc/manual/src/release-notes/rl-next.md | 5 +++++ src/nix/repl.cc | 16 +++------------- src/nix/repl.md | 2 +- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index efd893662..096499bc3 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -24,3 +24,8 @@ Selecting derivation outputs using the attribute selection syntax (e.g. `nixpkgs#glibc.dev`) no longer works. + +* `nix repl` now takes installables on the command line, unifying the usage + with other commands that use `--file` and `--expr`. Primary breaking change + is for the common usage of `nix repl ''` which can be recovered with + `nix repl nixpkgs` or `nix repl --expr 'import {}'` diff --git a/src/nix/repl.cc b/src/nix/repl.cc index b5ecc8ad0..ac0f1f4d7 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -73,7 +73,6 @@ struct NixRepl StorePath getDerivationPath(Value & v); bool processLine(std::string line); - void loadInstallable(Installable & installable); void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); void initEnv(); @@ -634,12 +633,6 @@ bool NixRepl::processLine(std::string line) return true; } -void NixRepl::loadInstallable(Installable & installable) -{ - auto [val, pos] = installable.toValue(*state); - addAttrsToScope(*val); -} - void NixRepl::loadFile(const Path & path) { loadedFiles.remove(path); @@ -899,6 +892,9 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m struct CmdRepl : InstallablesCommand { + CmdRepl(){ + evalSettings.pureEval = false; + } std::vector files; Strings getDefaultFlakeAttrPaths() override { @@ -908,10 +904,6 @@ struct CmdRepl : InstallablesCommand return file.has_value() or expr.has_value(); } - CmdRepl() - { - } - std::string description() override { return "start an interactive environment for evaluating Nix expressions"; @@ -926,8 +918,6 @@ struct CmdRepl : InstallablesCommand void run(ref store) override { - - evalSettings.pureEval = false; auto state = getEvalState(); auto repl = std::make_unique(searchPath, openStore(),state ,[&]()->NixRepl::AnnotatedValues{ diff --git a/src/nix/repl.md b/src/nix/repl.md index be1498e5b..6a526f7d0 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -48,7 +48,7 @@ R""( nix-repl> :q - # nix repl --expr 'import {}' --impure + # nix repl --expr 'import {}' Loading Installable ''... Added 12439 variables. From 9f8c1183fa10aa9d95bce0ca2f3337532ad7981b Mon Sep 17 00:00:00 2001 From: tomberek Date: Wed, 18 May 2022 21:18:07 -0400 Subject: [PATCH 006/119] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/nix/repl.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index ac0f1f4d7..a1b42b760 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -896,11 +896,12 @@ struct CmdRepl : InstallablesCommand evalSettings.pureEval = false; } std::vector files; - Strings getDefaultFlakeAttrPaths() - override { + Strings getDefaultFlakeAttrPaths() override + { return {""}; } - virtual bool useDefaultInstallables() { + virtual bool useDefaultInstallables() override + { return file.has_value() or expr.has_value(); } From 7534798eedb696226101f2c8793ba9ace049f5e4 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Wed, 18 May 2022 21:33:41 -0400 Subject: [PATCH 007/119] refactor: factor out getValue --- src/nix/repl.cc | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a1b42b760..cae76bb5d 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -920,18 +920,22 @@ struct CmdRepl : InstallablesCommand void run(ref store) override { auto state = getEvalState(); - auto repl = std::make_unique(searchPath, openStore(),state - ,[&]()->NixRepl::AnnotatedValues{ - auto installables = load(); - NixRepl::AnnotatedValues values; - for (auto & installable: installables){ - auto [val, pos] = installable->toValue(*state); - auto what = installable->what(); - values.push_back( {val,what} ); - } - return values; - } - ); + auto getValues = [&]()->NixRepl::AnnotatedValues{ + auto installables = load(); + NixRepl::AnnotatedValues values; + for (auto & installable: installables){ + auto [val, pos] = installable->toValue(*state); + auto what = installable->what(); + values.push_back( {val,what} ); + } + return values; + }; + auto repl = std::make_unique( + searchPath, + openStore(), + state, + getValues + ); repl->autoArgs = getAutoArgs(*repl->state); repl->mainLoop(); } From e1f308a1ec3c395cd4978b45400f7a45adcea0dc Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Wed, 18 May 2022 22:28:15 -0400 Subject: [PATCH 008/119] repl: provide backward compat with legacy usage --- src/libcmd/command.hh | 2 +- src/nix/repl.cc | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 65eb0c4a0..2c88e1526 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -120,7 +120,7 @@ struct InstallablesCommand : virtual Args, SourceExprCommand std::optional getFlakeRefForCompletion() override; -private: +protected: std::vector _installables; }; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index cae76bb5d..d4079816f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -895,6 +895,22 @@ struct CmdRepl : InstallablesCommand CmdRepl(){ evalSettings.pureEval = false; } + void prepare() + { + if (!settings.isExperimentalFeatureEnabled(Xp::Flakes) && !(file)) { + warn("future versions of Nix will require using `--file` to load a file"); + if (this->_installables.size() > 1) { + warn("more than one input file is not currently supported"); + } + if (this->_installables.size() >= 1) { + file = std::optional( + this->_installables[0].data() + ); + } + _installables.clear(); + } + installables = InstallablesCommand::load(); + } std::vector files; Strings getDefaultFlakeAttrPaths() override { From f21dec5befc9ee273a5210dec322d30c3c3be595 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Thu, 19 May 2022 01:01:45 -0400 Subject: [PATCH 009/119] repl: hide flake behavior behind flag and provide warning --- src/nix/repl.cc | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d4079816f..18cdb3580 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -897,17 +897,14 @@ struct CmdRepl : InstallablesCommand } void prepare() { - if (!settings.isExperimentalFeatureEnabled(Xp::Flakes) && !(file)) { + if (!settings.isExperimentalFeatureEnabled(Xp::Flakes) && !(file) && this->_installables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); - if (this->_installables.size() > 1) { + if (this->_installables.size() > 1) warn("more than one input file is not currently supported"); - } - if (this->_installables.size() >= 1) { - file = std::optional( - this->_installables[0].data() - ); - } - _installables.clear(); + auto filePath = this->_installables[0].data(); + file = std::optional(filePath); + _installables.front() = _installables.back(); + _installables.pop_back(); } installables = InstallablesCommand::load(); } @@ -940,9 +937,20 @@ struct CmdRepl : InstallablesCommand auto installables = load(); NixRepl::AnnotatedValues values; for (auto & installable: installables){ - auto [val, pos] = installable->toValue(*state); auto what = installable->what(); - values.push_back( {val,what} ); + if (!settings.isExperimentalFeatureEnabled(Xp::Flakes) && file){ + auto [val, pos] = installable->toValue(*state); + auto what = installable->what(); + state->forceValue(*val, pos); + auto autoArgs = getAutoArgs(*state); + Value *valPost = state->allocValue(); + state->autoCallFunction(*autoArgs, *val, *valPost); + state->forceValue(*valPost, pos); + values.push_back( {valPost, what }); + } else { + auto [val, pos] = installable->toValue(*state); + values.push_back( {val,what} ); + } } return values; }; From 7d7e00272a2f47f68b3809296992db84ae871e09 Mon Sep 17 00:00:00 2001 From: tomberek Date: Fri, 20 May 2022 01:28:20 -0400 Subject: [PATCH 010/119] Apply suggestions from code review Style fixes from @edolstra Co-authored-by: Eelco Dolstra --- src/nix/repl.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 18cdb3580..6c05daa11 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -97,7 +97,7 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store,ref state, +NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, std::function getValues) : state(state) , getValues(getValues) @@ -691,8 +691,8 @@ void NixRepl::reloadFiles() loadFile(i); } - for (auto & [i,what] : getValues()) { - notice("Loading Installable '%1%'...", what); + for (auto & [i, what] : getValues()) { + notice("Loading installable '%1%'...", what); addAttrsToScope(*i); } } @@ -943,13 +943,13 @@ struct CmdRepl : InstallablesCommand auto what = installable->what(); state->forceValue(*val, pos); auto autoArgs = getAutoArgs(*state); - Value *valPost = state->allocValue(); + auto valPost = state->allocValue(); state->autoCallFunction(*autoArgs, *val, *valPost); state->forceValue(*valPost, pos); values.push_back( {valPost, what }); } else { auto [val, pos] = installable->toValue(*state); - values.push_back( {val,what} ); + values.push_back( {val, what} ); } } return values; From db613a85fb7fb8c8a0f476f83db92523cce327f7 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 20 May 2022 01:35:06 -0400 Subject: [PATCH 011/119] repl: allow --file to always utilize autoargs --- src/nix/repl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 6c05daa11..e9898c08c 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -938,7 +938,7 @@ struct CmdRepl : InstallablesCommand NixRepl::AnnotatedValues values; for (auto & installable: installables){ auto what = installable->what(); - if (!settings.isExperimentalFeatureEnabled(Xp::Flakes) && file){ + if (file){ auto [val, pos] = installable->toValue(*state); auto what = installable->what(); state->forceValue(*val, pos); From 542e36c6e7ed5efa3d60e5adfc37ff7bb7e90a41 Mon Sep 17 00:00:00 2001 From: tomberek Date: Fri, 20 May 2022 01:48:24 -0400 Subject: [PATCH 012/119] Apply suggestions from code review Co-authored-by: Eelco Dolstra --- src/libcmd/installables.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b97888db6..948f78919 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -131,6 +131,7 @@ struct Installable OperateOn operateOn, const std::vector> & installables); }; + typedef std::vector> Installables; struct InstallableValue : Installable From 82c4af41e3348a87ebc9fb583df09070beadc019 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 20 May 2022 01:49:49 -0400 Subject: [PATCH 013/119] repl: clarify change and usage of --- doc/manual/src/release-notes/rl-next.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 096499bc3..6dbf6adae 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -28,4 +28,4 @@ * `nix repl` now takes installables on the command line, unifying the usage with other commands that use `--file` and `--expr`. Primary breaking change is for the common usage of `nix repl ''` which can be recovered with - `nix repl nixpkgs` or `nix repl --expr 'import {}'` + `nix repl --file ''` or `nix repl --expr 'import {}'` From 938150472d8373395f6f09cd76d4b0bde271ffda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 20 May 2022 08:12:02 +0200 Subject: [PATCH 014/119] Add some tests for the new REPL cli - Test that without the XP feature things work as before - Test that with or without the XP feature `--file file` works - Test that with XP feature passing a flakeref works - Test `:reload` with a flake --- tests/repl.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/tests/repl.sh b/tests/repl.sh index b6937b9e9..e879319fa 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -50,15 +50,17 @@ testRepl testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR" testReplResponse () { - local response="$(nix repl <<< "$1")" - echo "$response" | grep -qs "$2" \ + local commands="$1"; shift + local expectedResponse="$1"; shift + local response="$(nix repl "$@" <<< "$commands")" + echo "$response" | grep -qs "$expectedResponse" \ || fail "repl command set: -$1 +$commands does not respond with: -$2 +$expectedResponse but with: @@ -71,3 +73,48 @@ testReplResponse ' :a { a = "2"; } "result: ${a}" ' "result: 2" + +testReplResponse ' +drvPath +' '"/tmp/nix-test/default/store/qlksh7k4a72107vc054ilywq4rcmy9if-simple.drv"' \ +$testDir/simple.nix --experimental-features '' + +testReplResponse ' +drvPath +' '"/tmp/nix-test/default/store/qlksh7k4a72107vc054ilywq4rcmy9if-simple.drv"' \ +--file $testDir/simple.nix --experimental-features '' + +testReplResponse ' +drvPath +' '"/tmp/nix-test/default/store/qlksh7k4a72107vc054ilywq4rcmy9if-simple.drv"' \ +--file $testDir/simple.nix --experimental-features 'flakes' + +mkdir -p flake && cat < flake/flake.nix +{ + outputs = { self }: { + foo = 1; + bar.baz = 2; + + changingThing = "beforeChange"; + }; +} +EOF +testReplResponse ' +foo + baz +' "3" \ + ./flake ./flake\#bar + +# Test the `:reload` mechansim with flakes: +# - Eval `./flake#changingThing` +# - Modify the flake +# - Re-eval it +# - Check that the result has changed +replResult=$( ( +echo "changingThing" +sleep 1 # Leave the repl the time to eval 'foo' +sed -i 's/beforeChange/afterChange/' flake/flake.nix +echo ":reload" +echo "changingThing" +) | nix repl ./flake) +echo "$replResult" | grep -qs beforeChange +echo "$replResult" | grep -qs afterChange From 0053dab43f9ca350c27235f8a58b5d550bfffd38 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 20 May 2022 08:03:41 -0400 Subject: [PATCH 015/119] repl: fix tests to run on any testing store --- tests/repl.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/repl.sh b/tests/repl.sh index e879319fa..07f647585 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -76,17 +76,17 @@ testReplResponse ' testReplResponse ' drvPath -' '"/tmp/nix-test/default/store/qlksh7k4a72107vc054ilywq4rcmy9if-simple.drv"' \ +' '".*-simple.drv"' \ $testDir/simple.nix --experimental-features '' testReplResponse ' drvPath -' '"/tmp/nix-test/default/store/qlksh7k4a72107vc054ilywq4rcmy9if-simple.drv"' \ +' '".*-simple.drv"' \ --file $testDir/simple.nix --experimental-features '' testReplResponse ' drvPath -' '"/tmp/nix-test/default/store/qlksh7k4a72107vc054ilywq4rcmy9if-simple.drv"' \ +' '".*-simple.drv"' \ --file $testDir/simple.nix --experimental-features 'flakes' mkdir -p flake && cat < flake/flake.nix From 7a04fb1c56ca60652c2a44019b31fe8cf2e2bc46 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 20 May 2022 08:20:00 -0400 Subject: [PATCH 016/119] repl: add repl-flake experimental feature for gating --- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + src/nix/repl.cc | 2 +- tests/repl.sh | 6 +++--- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 315de64a4..fa79cca6b 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -13,6 +13,7 @@ std::map stringifiedXpFeatures = { { Xp::RecursiveNix, "recursive-nix" }, { Xp::NoUrlLiterals, "no-url-literals" }, { Xp::FetchClosure, "fetch-closure" }, + { Xp::ReplFlake, "repl-flake" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 57512830c..d09ab025c 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -22,6 +22,7 @@ enum struct ExperimentalFeature RecursiveNix, NoUrlLiterals, FetchClosure, + ReplFlake, }; /** diff --git a/src/nix/repl.cc b/src/nix/repl.cc index e9898c08c..b12f05c15 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -897,7 +897,7 @@ struct CmdRepl : InstallablesCommand } void prepare() { - if (!settings.isExperimentalFeatureEnabled(Xp::Flakes) && !(file) && this->_installables.size() >= 1) { + if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); if (this->_installables.size() > 1) warn("more than one input file is not currently supported"); diff --git a/tests/repl.sh b/tests/repl.sh index 07f647585..5caf0a58a 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -87,7 +87,7 @@ drvPath testReplResponse ' drvPath ' '".*-simple.drv"' \ ---file $testDir/simple.nix --experimental-features 'flakes' +--file $testDir/simple.nix --experimental-features 'repl-flake' mkdir -p flake && cat < flake/flake.nix { @@ -102,7 +102,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar + ./flake ./flake\#bar --experimental-features 'flakes repl-flake' # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` @@ -115,6 +115,6 @@ sleep 1 # Leave the repl the time to eval 'foo' sed -i 's/beforeChange/afterChange/' flake/flake.nix echo ":reload" echo "changingThing" -) | nix repl ./flake) +) | nix repl ./flake --experimental-features 'flakes repl-flake') echo "$replResult" | grep -qs beforeChange echo "$replResult" | grep -qs afterChange From 8c3939af14106c753bbb963663ad1cfb4fa6de80 Mon Sep 17 00:00:00 2001 From: tomberek Date: Fri, 20 May 2022 12:09:41 -0400 Subject: [PATCH 017/119] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/repl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/repl.sh b/tests/repl.sh index 5caf0a58a..e9e41558a 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -77,7 +77,7 @@ testReplResponse ' testReplResponse ' drvPath ' '".*-simple.drv"' \ -$testDir/simple.nix --experimental-features '' +$testDir/simple.nix testReplResponse ' drvPath @@ -87,7 +87,7 @@ drvPath testReplResponse ' drvPath ' '".*-simple.drv"' \ ---file $testDir/simple.nix --experimental-features 'repl-flake' +--file $testDir/simple.nix --extra-experimental-features 'repl-flake' mkdir -p flake && cat < flake/flake.nix { From 9151dbff88fa765496e970aee2db5a8ce640b3a4 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 2 Jun 2022 10:26:46 -0600 Subject: [PATCH 018/119] ignore-try flag --- src/libcmd/command.cc | 11 ++++++++++- src/libcmd/command.hh | 1 + src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 1 + src/libexpr/primops.cc | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 7f8072d75..940fd5b23 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -91,6 +91,12 @@ EvalCommand::EvalCommand() .description = "start an interactive environment if evaluation fails", .handler = {&startReplOnEvalErrors, true}, }); + + addFlag({ + .longName = "ignore-try", + .description = "ignore exceptions in try clauses during debug", + .handler = {&ignoreExceptionsDuringTry, true}, + }); } EvalCommand::~EvalCommand() @@ -120,7 +126,10 @@ ref EvalCommand::getEvalState() ; if (startReplOnEvalErrors) { - evalState->debugRepl = &runRepl; + evalState->debugRepl = &runRepl; + }; + if (ignoreExceptionsDuringTry) { + evalState->ignoreTry = ignoreExceptionsDuringTry; }; } return ref(evalState); diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 8982f21d0..2c930dcb7 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -58,6 +58,7 @@ struct CopyCommand : virtual StoreCommand struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; + bool ignoreExceptionsDuringTry = false; EvalCommand(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 40462afdf..c35527992 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -467,6 +467,7 @@ EvalState::EvalState( , debugRepl(0) , debugStop(false) , debugQuit(false) + , ignoreTry(false) , regexCache(makeRegexCache()) #if HAVE_BOEHMGC , valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7b8732169..3c3dddd1e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -130,6 +130,7 @@ public: void (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; bool debugQuit; + bool ignoreTry; std::list debugTraces; std::map> exprEnvs; const std::shared_ptr getStaticEnv(const Expr & expr) const diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index eea274301..772898932 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -851,6 +851,15 @@ static RegisterPrimOp primop_floor({ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto attrs = state.buildBindings(2); + + void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; + if (state.debugRepl && state.ignoreTry) + { + // to prevent starting the repl from exceptions withing a tryEval, null it. + savedDebugRepl = state.debugRepl; + state.debugRepl = nullptr; + } + try { state.forceValue(*args[0], pos); attrs.insert(state.sValue, args[0]); @@ -859,6 +868,11 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va attrs.alloc(state.sValue).mkBool(false); attrs.alloc("success").mkBool(false); } + + // restore the debugRepl pointer if we saved it earlier. + if (savedDebugRepl) + state.debugRepl = savedDebugRepl; + v.mkAttrs(attrs); } From bc0d41e9baa19c10977dd38f4bb255c14bd6554d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 2 Jun 2022 12:17:28 -0600 Subject: [PATCH 019/119] print message with exceptions in a try clause --- src/libexpr/eval.cc | 12 ++++++++++-- src/libexpr/eval.hh | 1 + src/libexpr/primops.cc | 15 +++++++++++---- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c35527992..60214453a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -464,10 +464,11 @@ EvalState::EvalState( , emptyBindings(0) , store(store) , buildStore(buildStore ? buildStore : store) - , debugRepl(0) + , debugRepl(nullptr) , debugStop(false) , debugQuit(false) , ignoreTry(false) + , trylevel(0) , regexCache(makeRegexCache()) #if HAVE_BOEHMGC , valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) @@ -833,7 +834,14 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & : nullptr; if (error) - printError("%s\n\n" ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL, error->what()); + { + printError("%s\n\n", error->what()); + + if (trylevel > 0 && error->info().level != lvlInfo) + printError("This exception occurred in a try clause. use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); + + printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL, error->what()); + } auto se = getStaticEnv(expr); if (se) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3c3dddd1e..9aff77042 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -131,6 +131,7 @@ public: bool debugStop; bool debugQuit; bool ignoreTry; + int trylevel; std::list debugTraces; std::map> exprEnvs; const std::shared_ptr getStaticEnv(const Expr & expr) const diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 772898932..e4fd8f650 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -853,11 +853,15 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va auto attrs = state.buildBindings(2); void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl && state.ignoreTry) + if (state.debugRepl) { - // to prevent starting the repl from exceptions withing a tryEval, null it. - savedDebugRepl = state.debugRepl; - state.debugRepl = nullptr; + state.trylevel++; + if (state.ignoreTry) + { + // to prevent starting the repl from exceptions withing a tryEval, null it. + savedDebugRepl = state.debugRepl; + state.debugRepl = nullptr; + } } try { @@ -873,6 +877,9 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va if (savedDebugRepl) state.debugRepl = savedDebugRepl; + if (state.debugRepl) + state.trylevel--; + v.mkAttrs(attrs); } From 8cf6ae86648336bd67a1555302b21576b790c368 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 2 Jun 2022 12:29:38 -0600 Subject: [PATCH 020/119] use Counter class to count tryEval levels --- src/libexpr/primops.cc | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e4fd8f650..ecc1c136a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -846,22 +846,30 @@ static RegisterPrimOp primop_floor({ .fun = prim_floor, }); +class Counter +{ + private: + int &counter; + public: + Counter(int &counter) :counter(counter) { counter++; } + ~Counter() { counter--; } +}; + /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto attrs = state.buildBindings(2); + /* increment state.trylevel, and decrement it when this function returns. */ + Counter trylevel(state.trylevel); + void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl) + if (state.debugRepl && state.ignoreTry) { - state.trylevel++; - if (state.ignoreTry) - { - // to prevent starting the repl from exceptions withing a tryEval, null it. - savedDebugRepl = state.debugRepl; - state.debugRepl = nullptr; - } + /* to prevent starting the repl from exceptions withing a tryEval, null it. */ + savedDebugRepl = state.debugRepl; + state.debugRepl = nullptr; } try { @@ -877,9 +885,6 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va if (savedDebugRepl) state.debugRepl = savedDebugRepl; - if (state.debugRepl) - state.trylevel--; - v.mkAttrs(attrs); } From 49ff4ef6373f4aaeb6194fb4a195d3037c74312e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 10 Jun 2022 12:22:36 -0600 Subject: [PATCH 021/119] remove unused parameter --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 60214453a..28256ec5c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -840,7 +840,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & if (trylevel > 0 && error->info().level != lvlInfo) printError("This exception occurred in a try clause. use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); - printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL, error->what()); + printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL); } auto se = getStaticEnv(expr); From 98946e2d9c93e3558f19ee3d49deef67a98706d8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 13 Jun 2022 23:01:13 +0200 Subject: [PATCH 022/119] nix-shell: restore backwards-compat with old nixpkgs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Basically an attempt to resume fixing #5543 for a breakage introduced earlier[1]. Basically, when evaluating an older `nixpkgs` with `nix-shell` the following error occurs: λ ma27 [~] → nix-shell -I nixpkgs=channel:nixos-18.03 -p nix error: anonymous function at /nix/store/zakqwc529rb6xcj8pwixjsxscvlx9fbi-source/pkgs/top-level/default.nix:20:1 called with unexpected argument 'inNixShell' at /nix/store/zakqwc529rb6xcj8pwixjsxscvlx9fbi-source/pkgs/top-level/impure.nix:82:1: 81| 82| import ./. (builtins.removeAttrs args [ "system" "platform" ] // { | ^ 83| inherit config overlays crossSystem; This is a problem because one of the main selling points of Nix is that you can evaluate any old Nix expression and still get the same result (which also means that it *still evaluates*). In fact we're deprecating, but not removing a lot of stuff for that reason such as unquoted URLs[2] or `builtins.toPath`. However this property was essentially thrown away here. The change is rather simple: check if `inNixShell` is specified in the formals of an auto-called function. This means that { inNixShell ? false }: builtins.trace inNixShell (with import { }; makeShell { name = "foo"; }) will show `trace: true` while args@{ ... }: builtins.trace args.inNixShell (with import { }; makeShell { name = "foo"; }) will throw the following error: error: attribute 'inNixShell' missing This is explicitly needed because the function in `pkgs/top-level/impure.nix` of e.g. NixOS 18.03 has an ellipsis[3], but passes the attribute-set on to another lambda with formals that doesn't have an ellipsis anymore (hence the error from above). This was perhaps a mistake, but we can't fix it anymore. This also means that there's AFAICS no proper way to check if the attr-set that's passed to the Nix code via `EvalState::autoCallFunction` is eventually passed to a lambda with formals where `inNixShell` is missing. However, this fix comes with a certain price. Essentially every `shell.nix` that assumes `inNixShell` to be passed to the formals even without explicitly specifying it would break with this[4]. However I think that this is ugly, but preferable: * Nix 2.3 was declared stable by NixOS up until recently (well, it still is as long as 21.11 is alive), so most people might not have even noticed that feature. * We're talking about a way shorter time-span with this change being in the wild, so the fallout should be smaller IMHO. [1] https://github.com/NixOS/nix/commit/9d612c393abc3a73590650d24bcfe2ee57792872 [2] https://github.com/NixOS/rfcs/pull/45#issuecomment-488232537 [3] https://github.com/NixOS/nixpkgs/blob/release-18.03/pkgs/top-level/impure.nix#L75 [4] See e.g. the second expression in this commit-message or the changes for `tests/ca/nix-shell.sh`. --- src/nix-build/nix-build.cc | 38 ++++++++++++++++++++++++++++++++++---- tests/ca-shell.nix | 2 +- tests/nix-shell.sh | 7 +++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 519855ea3..7eb8c8f6a 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -257,11 +257,12 @@ static void main_nix_build(int argc, char * * argv) auto autoArgs = myArgs.getAutoArgs(*state); + auto autoArgsWithInNixShell = autoArgs; if (runEnv) { - auto newArgs = state->buildBindings(autoArgs->size() + 1); + auto newArgs = state->buildBindings(autoArgsWithInNixShell->size() + 1); newArgs.alloc("inNixShell").mkBool(true); for (auto & i : *autoArgs) newArgs.insert(i); - autoArgs = newArgs.finish(); + autoArgsWithInNixShell = newArgs.finish(); } if (packages) { @@ -316,10 +317,39 @@ static void main_nix_build(int argc, char * * argv) Value vRoot; state->eval(e, vRoot); + std::function takesNixShellAttr; + takesNixShellAttr = [&](const Value & v) { + if (!runEnv) { + return false; + } + bool add = false; + if (v.type() == nFunction && v.lambda.fun->hasFormals()) { + for (auto & i : v.lambda.fun->formals->formals) { + if (state->symbols[i.name] == "inNixShell") { + add = true; + break; + } + } + } + return add; + }; + for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first); + Value & v(*findAlongAttrPath( + *state, + i, + takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs, + vRoot + ).first); state->forceValue(v, [&]() { return v.determinePos(noPos); }); - getDerivations(*state, v, "", *autoArgs, drvs, false); + getDerivations( + *state, + v, + "", + takesNixShellAttr(v) ? *autoArgsWithInNixShell : *autoArgs, + drvs, + false + ); } } diff --git a/tests/ca-shell.nix b/tests/ca-shell.nix index ad2ab6aff..36e1d1526 100644 --- a/tests/ca-shell.nix +++ b/tests/ca-shell.nix @@ -1 +1 @@ -{ ... }@args: import ./shell.nix (args // { contentAddressed = true; }) +{ inNixShell ? false, ... }@args: import ./shell.nix (args // { contentAddressed = true; }) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 3241d7a0f..0dc1999d8 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -102,3 +102,10 @@ source <(nix print-dev-env -f "$shellDotNix" shellDrv) [[ ${arr2[1]} = $'\n' ]] [[ ${arr2[2]} = $'x\ny' ]] [[ $(fun) = blabla ]] + +# Test nix-shell with ellipsis and no `inNixShell` argument (for backwards compat with old nixpkgs) +cat >$TEST_ROOT/shell-ellipsis.nix < Date: Wed, 15 Jun 2022 09:02:36 -0400 Subject: [PATCH 023/119] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix/repl.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 5749f3924..9684a70d4 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -7,3 +7,5 @@ with other commands that use `--file` and `--expr`. Primary breaking change is for the common usage of `nix repl ''` which can be recovered with `nix repl --file ''` or `nix repl --expr 'import {}'` + + This is currently guarded by the 'repl-flake' experimental feature diff --git a/src/nix/repl.md b/src/nix/repl.md index 6a526f7d0..23ef0f4e6 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,7 +36,7 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl nixpkgs + # nix repl --extra_experimental_features 'flakes repl-flake' nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. From 475249db8aa3c998de594b94b38e08b04b117a6c Mon Sep 17 00:00:00 2001 From: Lorenzo Manacorda Date: Wed, 15 Jun 2022 17:32:59 +0200 Subject: [PATCH 024/119] libstore: improve warning message on missing sig Clarifies that the substitute will be ignored/skipped. --- src/libstore/build/substitution-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index ca5218627..3a5da13fb 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -154,7 +154,7 @@ void PathSubstitutionGoal::tryNext() only after we've downloaded the path. */ if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { - warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'", + warn("igoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); tryNext(); return; From ca2be509b96a10a2035039a825fc2b292ec0ad4d Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Wed, 15 Jun 2022 16:38:56 -0400 Subject: [PATCH 025/119] Verify `$HOME` is owned by current user in `getHome()`, if it exists. Useful because a default `sudo` on darwin doesn't clear `$HOME`, so things like `sudo nix-channel --list` will surprisingly return the USER'S channels, rather than `root`'s. Other counterintuitive outcomes can be seen in this PR description: https://github.com/NixOS/nix/pull/6622 --- src/libutil/util.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1c19938a8..a368ac844 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -574,6 +574,20 @@ Path getHome() static Path homeDir = []() { auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("Couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir); + homeDir.reset(); + } + } if (!homeDir) { std::vector buf(16384); struct passwd pwbuf; From d6d0e781bbade76f6ea3f310cb36973f4013826d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sun, 19 Jun 2022 17:54:27 +0200 Subject: [PATCH 026/119] Complete flake inputs for all given flakes Allow `nix build flake1 flake2 --update-input ` to complete the inputs of both flakes. Also do tilde expansion so that `nix build ~/flake --update-input ` works. --- src/libcmd/command.hh | 10 ++++++---- src/libcmd/installables.cc | 34 ++++++++++++++++------------------ src/nix/flake.cc | 4 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 8982f21d0..cab379b84 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -79,8 +79,10 @@ struct MixFlakeOptions : virtual Args, EvalCommand MixFlakeOptions(); - virtual std::optional getFlakeRefForCompletion() + virtual std::vector getFlakesForCompletion() { return {}; } + + void completeFlakeInput(std::string_view prefix); }; struct SourceExprCommand : virtual Args, MixFlakeOptions @@ -119,7 +121,7 @@ struct InstallablesCommand : virtual Args, SourceExprCommand virtual bool useDefaultInstallables() { return true; } - std::optional getFlakeRefForCompletion() override; + std::vector getFlakesForCompletion() override; private: @@ -135,9 +137,9 @@ struct InstallableCommand : virtual Args, SourceExprCommand void prepare() override; - std::optional getFlakeRefForCompletion() override + std::vector getFlakesForCompletion() override { - return parseFlakeRefWithFragment(_installable, absPath(".")).first; + return {_installable}; } private: diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index ffc25135e..1bcef4172 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -23,15 +23,16 @@ namespace nix { -void completeFlakeInputPath( - ref evalState, - const FlakeRef & flakeRef, - std::string_view prefix) +void MixFlakeOptions::completeFlakeInput(std::string_view prefix) { - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); + auto evalState = getEvalState(); + for (auto & flakeRefS : getFlakesForCompletion()) { + auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions->add(input.first); + } } MixFlakeOptions::MixFlakeOptions() @@ -86,8 +87,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, .completer = {[&](size_t, std::string_view prefix) { - if (auto flakeRef = getFlakeRefForCompletion()) - completeFlakeInputPath(getEvalState(), *flakeRef, prefix); + completeFlakeInput(prefix); }} }); @@ -103,12 +103,10 @@ MixFlakeOptions::MixFlakeOptions() parseFlakeRef(flakeRef, absPath("."), true)); }}, .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) { - if (auto flakeRef = getFlakeRefForCompletion()) - completeFlakeInputPath(getEvalState(), *flakeRef, prefix); - } else if (n == 1) { + if (n == 0) + completeFlakeInput(prefix); + else if (n == 1) completeFlakeRef(getEvalState()->store, prefix); - } }} }); @@ -1043,14 +1041,14 @@ void InstallablesCommand::prepare() installables = parseInstallables(getStore(), _installables); } -std::optional InstallablesCommand::getFlakeRefForCompletion() +std::vector InstallablesCommand::getFlakesForCompletion() { if (_installables.empty()) { if (useDefaultInstallables()) - return parseFlakeRefWithFragment(".", absPath(".")).first; + return {"."}; return {}; } - return parseFlakeRefWithFragment(_installables.front(), absPath(".")).first; + return _installables; } InstallableCommand::InstallableCommand(bool supportReadOnlyMode) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8370b8dcf..439eb53ba 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -50,9 +50,9 @@ public: return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); } - std::optional getFlakeRefForCompletion() override + std::vector getFlakesForCompletion() override { - return getFlakeRef(); + return {flakeUrl}; } }; From 4ade8a5f25a32cdf591ea369318d9d256e49025a Mon Sep 17 00:00:00 2001 From: Fishhh Date: Mon, 20 Jun 2022 18:00:32 +0200 Subject: [PATCH 027/119] Fix arity of `--exclude` flag in `nix search` Due to incorrectly using the Handler(vector*) constructor the `--exclude` flag would swallow all proceeding arguments instead of just one. --- src/nix/search.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index f1f5f9641..bdd45cbed 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -34,7 +34,9 @@ struct CmdSearch : InstallableCommand, MixJSON .shortName = 'e', .description = "Hide packages whose attribute path, name or description contain *regex*.", .labels = {"regex"}, - .handler = Handler(&excludeRes), + .handler = {[this](std::string s) { + excludeRes.push_back(s); + }}, }); } From df21173b70a21ca679e644b55f3539c48167d92c Mon Sep 17 00:00:00 2001 From: Fishhh Date: Mon, 20 Jun 2022 18:29:18 +0200 Subject: [PATCH 028/119] Add another test for `--exclude` in `nix search` --- tests/search.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/search.sh b/tests/search.sh index 41b706ac6..1a98f5b49 100644 --- a/tests/search.sh +++ b/tests/search.sh @@ -43,3 +43,4 @@ e=$'\x1b' # grep doesn't support \e, \033 or even \x1b (( $(nix search -f search.nix foo --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) (( $(nix search -f search.nix foo -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) +[[ $(nix search -f search.nix -e bar --json | jq -c 'keys') == '["foo","hello"]' ]] From 983efdbde47bd0ecaff866d43c3155761574c112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 21 Jun 2022 14:08:18 +0200 Subject: [PATCH 029/119] Forbid the tilde expansion in pure eval mode Fix #6684 --- src/libexpr/parser.y | 6 ++++++ tests/pure-eval.sh | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 8cbc2da4d..7c9b5a2db 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -520,6 +520,12 @@ path_start $$ = new ExprPath(path); } | HPATH { + if (evalSettings.pureEval) { + throw Error( + "the path '%s' can not be resolved in pure mode", + std::string_view($1.p, $1.l) + ); + } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); $$ = new ExprPath(path); } diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 1a4568ea6..b83ab8afe 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -30,3 +30,5 @@ nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + " rm -rf $TEST_ROOT/eval-out (! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }') + +(! nix eval --expr '~/foo') From 3a85fd077cf8a404e0b7c727e47b8fcee85280a5 Mon Sep 17 00:00:00 2001 From: "Manu [tennox]" <2084639+tennox@users.noreply.github.com> Date: Tue, 21 Jun 2022 17:26:32 +0100 Subject: [PATCH 030/119] #6542 Apply flake templates partially on conflicts Will still exit with non-zero exit code and clearly prompt which files to merge: ``` nixx flake init -t github:numtide/devshell wrote: /home/manu/dev/stuff/gopassbridge/.envrc refusing to overwrite existing file '/home/manu/dev/stuff/gopassbridge/.gitignore' -> merge manually with '/nix/store/ksmwhyghjwb4d9dw6hcpbvng1msdvjim-source/template/.gitignore' wrote: /home/manu/dev/stuff/gopassbridge/devshell.toml wrote: /home/manu/dev/stuff/gopassbridge/flake.nix error: Encountered 1 conflicts - please merge manually ``` --- src/nix/flake.cc | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a1edb5dbf..24255c247 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -740,7 +740,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand "If you've set '%s' to a string, try using a path instead.", templateDir, templateDirAttr->getAttrPathStr()); - std::vector files; + std::vector changedFiles; + std::vector conflictedFiles; + auto success = false; std::function copyDir; copyDir = [&](const Path & from, const Path & to) @@ -757,22 +759,33 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto contents = readFile(from2); if (pathExists(to2)) { auto contents2 = readFile(to2); - if (contents != contents2) - throw Error("refusing to overwrite existing file '%s' - please merge manually with '%s'", to2, from2); + if (contents != contents2) { + printError("refusing to overwrite existing file '%s'\n-> merge manually with '%s'", to2, from2); + success = false; + conflictedFiles.push_back(to2); + } else { + notice("skipping identical file: %s", from2); + } + continue; } else writeFile(to2, contents); } else if (S_ISLNK(st.st_mode)) { auto target = readLink(from2); if (pathExists(to2)) { - if (readLink(to2) != target) - throw Error("refusing to overwrite existing symlink '%s' - please merge manually with '%s'", to2, from2); + if (readLink(to2) != target) { + printError("refusing to overwrite existing file '%s' - please merge manually with '%s'", to2, from2); + success = false; + conflictedFiles.push_back(to2); + } else { + notice("skipping identical file: %s", from2); + } } else createSymlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); - files.push_back(to2); + changedFiles.push_back(to2); notice("wrote: %s", to2); } }; @@ -781,7 +794,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand if (pathExists(flakeDir + "/.git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; - for (auto & s : files) args.push_back(s); + for (auto & s : changedFiles) args.push_back(s); runProgram("git", true, args); } auto welcomeText = cursor->maybeGetAttr("welcomeText"); @@ -789,6 +802,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand notice("\n"); notice(renderMarkdownToTerminal(welcomeText->getString())); } + + if (!success) + throw Error("Encountered %d conflicts - please merge manually", conflictedFiles.size()); } }; From f6cf644e5f7da4a0391b10fb31b4b4661c5439dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 15:35:52 +0200 Subject: [PATCH 031/119] Style --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a368ac844..aabd23427 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -580,7 +580,7 @@ Path getHome() int result = stat(homeDir->c_str(), &st); if (result != 0) { if (errno != ENOENT) { - warn("Couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); homeDir.reset(); } } else if (st.st_uid != geteuid()) { From d533a885465846e7512ff976d3599685c90316eb Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Wed, 22 Jun 2022 10:49:18 -0400 Subject: [PATCH 032/119] nar-info-disk-cache: refresh nix-cache-info weekly This allows changes to nix-cache-info to be picked up by existing clients. Previously, the only way for this to happen would be for clients to delete binary-cache-v6.sqlite, which is quite awkward for users. On the other hand, updates to nix-cache-info should be pretty rare, hence the choice of a fairly long TTL. Configurability is probably not useful enough to warrant implementing it. --- src/libstore/nar-info-disk-cache.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 9dd81ddfb..00325fcb8 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -62,6 +62,9 @@ public: /* How often to purge expired entries from the cache. */ const int purgeInterval = 24 * 3600; + /* How long to cache binary cache info (i.e. /nix-cache-info) */ + const int cacheInfoTtl = 7 * 24 * 3600; + struct Cache { int id; @@ -98,7 +101,7 @@ public: "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); state->queryCache.create(state->db, - "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + "select id, timestamp, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " @@ -186,8 +189,11 @@ public: auto queryCache(state->queryCache.use()(uri)); if (!queryCache.next()) return std::nullopt; + if (queryCache.getInt(1) + cacheInfoTtl < time(0)) + return std::nullopt; + state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); + Cache{(int) queryCache.getInt(0), queryCache.getStr(2), queryCache.getInt(3) != 0, (int) queryCache.getInt(4)}); } auto & cache(getCache(*state, uri)); From 696121fe1d110764bf4ca7f77c2ea57d0c1d3122 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 17:53:29 +0200 Subject: [PATCH 033/119] Fix incremental static builds $? refers to the object files that are newer, so the resulting file would lack all the older object files. --- mk/libraries.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/libraries.mk b/mk/libraries.mk index 876148a55..6541775f3 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -125,7 +125,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - +$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$? + +$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$^ $$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) From e94aa1f6473196f04c339f8661ee4df7b5a0d3be Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 22 Jun 2022 22:35:48 +0200 Subject: [PATCH 034/119] tests/nix-shell: more meaningful testcase --- tests/nix-shell.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 0dc1999d8..f291c6f79 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -105,7 +105,8 @@ source <(nix print-dev-env -f "$shellDotNix" shellDrv) # Test nix-shell with ellipsis and no `inNixShell` argument (for backwards compat with old nixpkgs) cat >$TEST_ROOT/shell-ellipsis.nix < Date: Wed, 22 Jun 2022 22:41:14 +0200 Subject: [PATCH 035/119] Enable/fix tests in nix-static pkgsStatic is apparently considered a cross environment, so checkPhase and installCheckPhase are disabled even when we ask for them. --- flake.nix | 11 ++++++++--- tests/ca/content-addressed.nix | 2 +- tests/common.sh.in | 2 ++ tests/fmt.sh | 7 ++++++- tests/local.mk | 6 +++++- tests/plugins.sh | 5 +++++ 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/flake.nix b/flake.nix index d683570af..e065061a8 100644 --- a/flake.nix +++ b/flake.nix @@ -571,14 +571,19 @@ nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps ++ propagatedDeps; + # Work around pkgsStatic disabling all tests. + preHook = + '' + doCheck=1 + doInstallCheck=1 + ''; + configureFlags = [ "--sysconfdir=/etc" ]; enableParallelBuilding = true; makeFlags = "profiledir=$(out)/etc/profile.d"; - doCheck = true; - installFlags = "sysconfdir=$(out)/etc"; postInstall = '' @@ -588,7 +593,6 @@ echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products ''; - doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; stripAllList = ["bin"]; @@ -597,6 +601,7 @@ hardeningDisable = [ "pie" ]; }; + dockerImage = let pkgs = nixpkgsFor.${system}; diff --git a/tests/ca/content-addressed.nix b/tests/ca/content-addressed.nix index 31c144ae0..81bc4bf5c 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -75,7 +75,7 @@ rec { buildCommand = '' mkdir -p $out/bin echo ${rootCA} # Just to make it depend on it - echo "" > $out/bin/${name} + echo "#! ${shell}" > $out/bin/${name} chmod +x $out/bin/${name} ''; }; diff --git a/tests/common.sh.in b/tests/common.sh.in index 6cb579e0d..5efd025ee 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -50,6 +50,8 @@ export busybox="@sandbox_shell@" export version=@PACKAGE_VERSION@ export system=@system@ +export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@ + export IMPURE_VAR1=foo export IMPURE_VAR2=bar diff --git a/tests/fmt.sh b/tests/fmt.sh index bc05118ff..254681ca2 100644 --- a/tests/fmt.sh +++ b/tests/fmt.sh @@ -18,7 +18,12 @@ cat << EOF > flake.nix with import ./config.nix; mkDerivation { name = "formatter"; - buildCommand = "mkdir -p \$out/bin; cp \${./fmt.simple.sh} \$out/bin/formatter"; + buildCommand = '' + mkdir -p \$out/bin + echo "#! ${shell}" > \$out/bin/formatter + cat \${./fmt.simple.sh} >> \$out/bin/formatter + chmod +x \$out/bin/formatter + ''; }; }; } diff --git a/tests/local.mk b/tests/local.mk index 2932d2b13..ae15c70f9 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -114,4 +114,8 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix -test-deps += tests/common.sh tests/config.nix tests/ca/config.nix tests/plugins/libplugintest.$(SO_EXT) +test-deps += tests/common.sh tests/config.nix tests/ca/config.nix + +ifeq ($(BUILD_SHARED_LIBS), 1) + test-deps += tests/plugins/libplugintest.$(SO_EXT) +endif diff --git a/tests/plugins.sh b/tests/plugins.sh index e22bf4408..6e278ad9d 100644 --- a/tests/plugins.sh +++ b/tests/plugins.sh @@ -2,6 +2,11 @@ source common.sh set -o pipefail +if [[ $BUILD_SHARED_LIBS != 1 ]]; then + echo "plugins are not supported" + exit 99 +fi + res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) [ "$res"x = "nullx" ] From 155c57c17131770a33dbd86055684d3605a0d505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 22 Jun 2022 11:24:20 +0200 Subject: [PATCH 036/119] nix develop: save XDG_DATA_DIRS for loadable completion --- src/nix/develop.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 2a3fc0213..6d9ad9942 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -276,15 +276,25 @@ struct Common : InstallableCommand, MixProfile const BuildEnvironment & buildEnvironment, const Path & outputsDir = absPath(".") + "/outputs") { + // A list of colon-separated environment variables that should be + // prepended to, rather than overwritten, in order to keep the shell usable. + // Please keep this list minimal in order to avoid impurities. + static const char * const savedVars[] = { + "PATH", // for commands + "XDG_DATA_DIRS", // for loadable completion + }; + std::ostringstream out; out << "unset shellHook\n"; - out << "nix_saved_PATH=\"$PATH\"\n"; + for (auto & var : savedVars) + out << fmt("nix_saved_%s=\"$%s\"\n", var, var); buildEnvironment.toBash(out, ignoreVars); - out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + for (auto & var : savedVars) + out << fmt("%s=\"$%s:$nix_saved_%s\"\n", var, var, var); out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) From d3176ce076407ef3e63667c0436bccf8be317ae4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 22:43:53 +0200 Subject: [PATCH 037/119] Fix build-remote in nix-static 'build-remote' is now executed via /proc/self/exe so it always works. --- src/libstore/build/hook-instance.cc | 25 ++++++++++++++++++------- src/libstore/globals.cc | 5 +++-- src/libstore/globals.hh | 2 +- src/libutil/util.cc | 14 ++++++++++++++ src/libutil/util.hh | 6 +++++- src/nix/main.cc | 5 +++++ src/nix/run.cc | 2 +- 7 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 0f6f580be..1f19ddccc 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -7,6 +7,22 @@ HookInstance::HookInstance() { debug("starting build hook '%s'", settings.buildHook); + auto buildHookArgs = tokenizeString>(settings.buildHook.get()); + + if (buildHookArgs.empty()) + throw Error("'build-hook' setting is empty"); + + auto buildHook = buildHookArgs.front(); + buildHookArgs.pop_front(); + + Strings args; + + for (auto & arg : buildHookArgs) + args.push_back(arg); + + args.push_back(std::string(baseNameOf(settings.buildHook.get()))); + args.push_back(std::to_string(verbosity)); + /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -36,14 +52,9 @@ HookInstance::HookInstance() if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; + execv(buildHook.c_str(), stringsToCharPtrs(args).data()); - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing '%s'", settings.buildHook); + throw SysError("executing '%s'", buildHook); }); pid.setSeparatePG(true); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index cc009a026..1d7f65135 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -67,12 +67,13 @@ Settings::Settings() sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); #endif - -/* chroot-like behavior from Apple's sandbox */ + /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif + + buildHook = getSelfExe().value_or("nix") + " __build-remote"; } void loadConfFile() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0ee27ecb6..9df1c999c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -195,7 +195,7 @@ public: )", {"build-timeout"}}; - PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + PathSetting buildHook{this, true, "", "build-hook", "The path of the helper program that executes builds to remote machines."}; Setting builders{ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index aabd23427..82628461c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -633,6 +633,20 @@ Path getDataDir() } +std::optional getSelfExe() +{ + static std::optional cached = []() + { + #if __linux__ + return readLink("/proc/self/exe"); + #else + return std::nullopt; + #endif + }(); + return cached; +} + + Paths createDirs(const Path & path) { Paths created; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 90418b04d..d3ed15b0b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -149,10 +149,14 @@ std::vector getConfigDirs(); /* Return $XDG_DATA_HOME or $HOME/.local/share. */ Path getDataDir(); +/* Return the path of the current executable. */ +std::optional getSelfExe(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) { +inline Paths createDirs(PathView path) +{ return createDirs(Path(path)); } diff --git a/src/nix/main.cc b/src/nix/main.cc index f398e3118..17c92ebc6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -266,6 +266,11 @@ void mainWrapped(int argc, char * * argv) programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); + if (argc > 0 && std::string_view(argv[0]) == "__build-remote") { + programName = "build-remote"; + argv++; argc--; + } + { auto legacy = (*RegisterLegacyCommand::commands)[programName]; if (legacy) return legacy(argc, argv); diff --git a/src/nix/run.cc b/src/nix/run.cc index 25a8fa8d3..45d2dfd0d 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -47,7 +47,7 @@ void runProgramInStore(ref store, Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; for (auto & arg : args) helperArgs.push_back(arg); - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + execv(getSelfExe().value_or("nix").c_str(), stringsToCharPtrs(helperArgs).data()); throw SysError("could not execute chroot helper"); } From 184f4e40de0960deccad2147099ea232e5e036c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 23:45:36 +0200 Subject: [PATCH 038/119] Remove NIX_LIBEXEC_DIR --- src/libstore/globals.cc | 1 - src/libstore/globals.hh | 3 --- src/libstore/local.mk | 1 - 3 files changed, 5 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1d7f65135..0f2ca4b15 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -36,7 +36,6 @@ Settings::Settings() , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9df1c999c..d7f351166 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -79,9 +79,6 @@ public: /* A list of user configuration files to load. */ std::vector nixUserConfFiles; - /* The directory where internal helper programs are stored. */ - Path nixLibexecDir; - /* The directory where the main programs are stored. */ Path nixBinDir; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b992bcbc0..0f94d3917 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -39,7 +39,6 @@ libstore_CXXFLAGS += \ -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ - -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ -DNIX_MAN_DIR=\"$(mandir)\" \ -DLSOF=\"$(lsof)\" From 1e55ee2961eabd6016dfef1793996ded97c9054c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 01:32:17 +0200 Subject: [PATCH 039/119] getSelfExe(): Support macOS --- src/libutil/util.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 82628461c..28df30fef 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -29,6 +29,7 @@ #ifdef __APPLE__ #include +#include #endif #ifdef __linux__ @@ -635,10 +636,17 @@ Path getDataDir() std::optional getSelfExe() { - static std::optional cached = []() + static auto cached = []() -> std::optional { #if __linux__ return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; #else return std::nullopt; #endif From 925b97522497e9c0f7a385c904410e560796208f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 18:21:37 +0200 Subject: [PATCH 040/119] Embed the sandbox shell into the statically linked 'nix' binary With this, Nix will write a copy of the sandbox shell to /bin/sh in the sandbox rather than bind-mounting it from the host filesystem. This makes /bin/sh work out of the box with nix-static, i.e. you no longer get /nix/store/qa36xhc5gpf42l3z1a8m1lysi40l9p7s-bootstrap-stage4-stdenv-linux/setup: ./configure: /bin/sh: bad interpreter: No such file or directory --- Makefile.config.in | 5 +++-- configure.ac | 8 ++++++++ flake.nix | 6 +++++- src/libstore/build/local-derivation-goal.cc | 14 +++++++++++++- src/libstore/local.mk | 10 ++++++++++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index d724853fa..1c5405c6d 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,4 +1,3 @@ -HOST_OS = @host_os@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ BOOST_LDFLAGS = @BOOST_LDFLAGS@ @@ -13,13 +12,14 @@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ +HOST_OS = @host_os@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ LOWDOWN_LIBS = @LOWDOWN_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ -LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SHELL = @bash@ @@ -31,6 +31,7 @@ datadir = @datadir@ datarootdir = @datarootdir@ doc_generate = @doc_generate@ docdir = @docdir@ +embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ includedir = @includedir@ libdir = @libdir@ diff --git a/configure.ac b/configure.ac index 15d5606c9..f0210ab78 100644 --- a/configure.ac +++ b/configure.ac @@ -320,6 +320,14 @@ if test ${cross_compiling:-no} = no && ! test -z ${sandbox_shell+x}; then fi fi +AC_ARG_ENABLE(embedded-sandbox-shell, AS_HELP_STRING([--enable-embedded-sandbox-shell],[include the sandbox shell in the Nix binary [default=no]]), + embedded_sandbox_shell=$enableval, embedded_sandbox_shell=no) +AC_SUBST(embedded_sandbox_shell) +if test "$embedded_sandbox_shell" = yes; then + AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.]) +fi + + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix test "$exec_prefix" = NONE && exec_prefix='${prefix}' diff --git a/flake.nix b/flake.nix index e065061a8..5df593940 100644 --- a/flake.nix +++ b/flake.nix @@ -578,7 +578,11 @@ doInstallCheck=1 ''; - configureFlags = [ "--sysconfdir=/etc" ]; + configureFlags = + configureFlags ++ + [ "--sysconfdir=/etc" + "--enable-embedded-sandbox-shell" + ]; enableParallelBuilding = true; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3ac9c20f9..d1ec91ed5 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1717,7 +1717,19 @@ void LocalDerivationGoal::runChild() for (auto & i : dirsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + + #if HAVE_EMBEDDED_SANDBOX_SHELL + if (i.second.source == "__embedded_sandbox_shell__") { + static unsigned char sh[] = { + #include "embedded-sandbox-shell.gen.hh" + }; + auto dst = chrootRootDir + i.first; + createDirs(dirOf(dst)); + writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); + chmod_(dst, 0555); + } else + #endif + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); } /* Bind a new instance of procfs on /proc. */ diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b992bcbc0..6f05c0d44 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -44,9 +44,19 @@ libstore_CXXFLAGS += \ -DNIX_MAN_DIR=\"$(mandir)\" \ -DLSOF=\"$(lsof)\" +ifeq ($(embedded_sandbox_shell),yes) +libstore_CXXFLAGS += -DSANDBOX_SHELL=\"__embedded_sandbox_shell__\" + +$(d)/build/local-derivation-goal.cc: $(d)/embedded-sandbox-shell.gen.hh + +$(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell) + $(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp + @mv $@.tmp $@ +else ifneq ($(sandbox_shell),) libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" endif +endif $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh From 0b2ea0023c81e8631df549dce996833ab5213a4a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 14:22:11 +0200 Subject: [PATCH 041/119] Fix typo --- src/libstore/build/substitution-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 3a5da13fb..2af105b4d 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -154,7 +154,7 @@ void PathSubstitutionGoal::tryNext() only after we've downloaded the path. */ if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { - warn("igoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); tryNext(); return; From 2a9fddc0b16d9b4771d11fc10d8b2a9cba55ff64 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 16:29:50 +0200 Subject: [PATCH 042/119] Automatically use a chroot store if /nix doesn't exist Specifically, if we're not root and the daemon socket does not exist, then we use ~/.local/share/nix/root as a chroot store. This enables non-root users to download nix-static and have it work out of the box, e.g. ubuntu@ip-10-13-1-146:~$ ~/nix run nixpkgs#hello warning: '/nix' does not exists, so Nix will use '/home/ubuntu/.local/share/nix/root' as a chroot store Hello, world! --- src/libstore/store-api.cc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8861274a2..b46b3066b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1302,7 +1302,8 @@ std::pair splitUriAndParams(const std::string & uri_ return {uri, params}; } -static bool isNonUriPath(const std::string & spec) { +static bool isNonUriPath(const std::string & spec) +{ return // is not a URL spec.find("://") == std::string::npos @@ -1319,7 +1320,19 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); - else + else if (!pathExists(stateDir) && params.empty() && getuid() != 0) { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/nix/root"; + if (!pathExists(chrootStore)) + warn("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + else + debug("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + Store::Params params2; + params2["root"] = chrootStore; + return std::make_shared(params2); + } else return std::make_shared(params); } else if (uri == "daemon") { return std::make_shared(params); From 1cb376d60e3a7d0742d92fa2ea1ebebba0a513e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 17:18:22 +0200 Subject: [PATCH 043/119] Fix typo Co-authored-by: Cole Helbling --- src/libstore/store-api.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b46b3066b..91080a2af 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1326,9 +1326,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para store in ~/.local/share/nix/root. */ auto chrootStore = getDataDir() + "/nix/root"; if (!pathExists(chrootStore)) - warn("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + warn("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); else - debug("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + debug("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); Store::Params params2; params2["root"] = chrootStore; return std::make_shared(params2); From 561a258f1d9fd11a5e111e14c492ee166a7551c1 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 23 Jun 2022 14:24:23 -0400 Subject: [PATCH 044/119] libstore/nar-info: drop unused system field This was unused everywhere (and even the official NixOS binary cache did not produce .narinfo files containing a "System:" field). --- src/libstore/nar-info.cc | 5 ----- src/libstore/nar-info.hh | 1 - 2 files changed, 6 deletions(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 2d75e7a82..071d8355e 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -69,8 +69,6 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (value != "unknown-deriver") deriver = StorePath(value); } - else if (name == "System") - system = value; else if (name == "Sig") sigs.insert(value); else if (name == "CA") { @@ -106,9 +104,6 @@ std::string NarInfo::to_string(const Store & store) const if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; - if (!system.empty()) - res += "System: " + system + "\n"; - for (auto sig : sigs) res += "Sig: " + sig + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..01683ec73 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -14,7 +14,6 @@ struct NarInfo : ValidPathInfo std::string compression; std::optional fileHash; uint64_t fileSize = 0; - std::string system; NarInfo() = delete; NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } From 8cf26385cd8c0e33e36f8d95b9224160424c1c60 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 23 Jun 2022 14:52:16 -0400 Subject: [PATCH 045/119] [fixup] handle cache expiration in sqlite query --- src/libstore/nar-info-disk-cache.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 00325fcb8..f4ea739b0 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -101,7 +101,7 @@ public: "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); state->queryCache.create(state->db, - "select id, timestamp, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " @@ -186,14 +186,11 @@ public: auto i = state->caches.find(uri); if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)); + auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl)); if (!queryCache.next()) return std::nullopt; - if (queryCache.getInt(1) + cacheInfoTtl < time(0)) - return std::nullopt; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(2), queryCache.getInt(3) != 0, (int) queryCache.getInt(4)}); + Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); } auto & cache(getCache(*state, uri)); From 2beb929753d28604ccd40057fca295a11640e40e Mon Sep 17 00:00:00 2001 From: Rick van Schijndel Date: Thu, 23 Jun 2022 21:11:08 +0200 Subject: [PATCH 046/119] eval-cache: cast rowId to correct type Prevents errors when running with UBSan: /nix/store/j5vhrywqmz1ixwhsmmjjxa85fpwryzh0-gcc-11.3.0/include/c++/11.3.0/bits/stl_pair.h:353:4: runtime error: load of value 229, which is not a valid value for type 'AttrType' --- src/libexpr/eval-cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index d77b25898..dbfd8e70b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -282,7 +282,7 @@ struct AttrDb auto queryAttribute(state->queryAttribute.use()(key.first)(symbols[key.second])); if (!queryAttribute.next()) return {}; - auto rowId = (AttrType) queryAttribute.getInt(0); + auto rowId = (AttrId) queryAttribute.getInt(0); auto type = (AttrType) queryAttribute.getInt(1); switch (type) { From 4b6cc3da62ba33c6861a3aa85353d75e8ac82464 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 23:56:26 +0200 Subject: [PATCH 047/119] Fetch flake-registry.json from channels.nixos.org Using fastly is slightly faster, provides some resilience due to a high stale TTL, and allows some usage metrics. --- src/libfetchers/fetch-settings.hh | 2 +- tests/github-flakes.nix | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 04c9feda0..6452143a1 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -70,7 +70,7 @@ struct FetchSettings : public Config Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; - Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; Setting useRegistries{this, true, "use-registries", diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index ddae6a21c..fc481c7e3 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -7,7 +7,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { let - # Generate a fake root CA and a fake github.com certificate. + # Generate a fake root CA and a fake api.github.com / channels.nixos.org certificate. cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } '' mkdir -p $out @@ -18,7 +18,7 @@ let openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr - openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:raw.githubusercontent.com") \ + openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:channels.nixos.org") \ -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt ''; @@ -67,7 +67,7 @@ makeTest ( name = "github-flakes"; nodes = - { # Impersonate github.com and api.github.com. + { github = { config, pkgs, ... }: { networking.firewall.allowedTCPPorts = [ 80 443 ]; @@ -77,12 +77,12 @@ makeTest ( services.httpd.extraConfig = '' ErrorLog syslog:local6 ''; - services.httpd.virtualHosts."github.com" = + services.httpd.virtualHosts."channels.nixos.org" = { forceSSL = true; sslServerKey = "${cert}/server.key"; sslServerCert = "${cert}/server.crt"; servedDirs = - [ { urlPath = "/NixOS/flake-registry/raw/master"; + [ { urlPath = "/"; dir = registry; } ]; @@ -109,7 +109,7 @@ makeTest ( nix.extraOptions = "experimental-features = nix-command flakes"; environment.systemPackages = [ pkgs.jq ]; networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = - [ "github.com" "api.github.com" "raw.githubusercontent.com" ]; + [ "channels.nixos.org" "api.github.com" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; }; @@ -123,7 +123,7 @@ makeTest ( github.wait_for_unit("httpd.service") - client.succeed("curl -v https://github.com/ >&2") + client.succeed("curl -v https://api.github.com/ >&2") client.succeed("nix registry list | grep nixpkgs") rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") From c6f7726f48e83230246f9328115368547fe29f5f Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Wed, 15 Jun 2022 12:49:12 -0400 Subject: [PATCH 048/119] Don't capture stdout when launching subshells in `nix repl` --- src/libcmd/repl.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 3c89a8ea3..588115a48 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -111,23 +111,20 @@ NixRepl::~NixRepl() write_history(historyFile.c_str()); } -std::string runNix(Path program, const Strings & args, +void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - auto res = runProgram(RunOptions { + runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, }); - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; + return; } static NixRepl * curRepl; // ugly From f801d70ba70c130a26747aa5b60d233f37d34bfa Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 24 Jun 2022 11:17:29 -0400 Subject: [PATCH 049/119] tests: enable ca-derivations for simple.nix in repl tests --- tests/repl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/repl.sh b/tests/repl.sh index 30921af04..c555560cc 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -87,12 +87,12 @@ $testDir/simple.nix testReplResponse ' drvPath ' '".*-simple.drv"' \ ---file $testDir/simple.nix --experimental-features '' +--file $testDir/simple.nix --experimental-features 'ca-derivations' testReplResponse ' drvPath ' '".*-simple.drv"' \ ---file $testDir/simple.nix --extra-experimental-features 'repl-flake' +--file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' mkdir -p flake && cat < flake/flake.nix { From 749d914d10e7550fb26a3a1599e2d4d7ed59b00a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jun 2022 23:14:56 +0200 Subject: [PATCH 050/119] Add reminder to remove nix-static testing hack https://github.com/NixOS/nix/pull/6708#issuecomment-1165912951 --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 5df593940..22367df71 100644 --- a/flake.nix +++ b/flake.nix @@ -572,6 +572,7 @@ buildInputs = buildDeps ++ propagatedDeps; # Work around pkgsStatic disabling all tests. + # Remove in NixOS 22.11, see https://github.com/NixOS/nixpkgs/pull/140271. preHook = '' doCheck=1 From 30d4aa5dd651813578b67d70ffbcd0446f6f0fe7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jun 2022 23:35:21 +0200 Subject: [PATCH 051/119] Only do the auto chroot store on Linux --- src/libstore/store-api.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 91080a2af..53b1a8777 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1320,6 +1320,7 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); + #if __linux__ else if (!pathExists(stateDir) && params.empty() && getuid() != 0) { /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot @@ -1332,7 +1333,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para Store::Params params2; params2["root"] = chrootStore; return std::make_shared(params2); - } else + } + #endif + else return std::make_shared(params); } else if (uri == "daemon") { return std::make_shared(params); From e8109cf405d672c50b1e5a25c632ddcb1d517233 Mon Sep 17 00:00:00 2001 From: Guillaume Girol Date: Sun, 26 Jun 2022 12:00:00 +0000 Subject: [PATCH 052/119] fetchGit: document `shallow` argument --- src/libexpr/primops/fetchTree.cc | 4 ++++ src/libfetchers/git.cc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e5eeea520..84e7f5c02 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -364,6 +364,10 @@ static RegisterPrimOp primop_fetchGit({ A Boolean parameter that specifies whether submodules should be checked out. Defaults to `false`. + - shallow\ + A Boolean parameter that specifies whether fetching a shallow clone + is allowed. Defaults to `false`. + - allRefs\ Whether to fetch all refs of the repository. With this argument being true, it's possible to load a `rev` from *any* `ref` (by default only diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 35fdf807a..7d01aaa7a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -574,7 +574,7 @@ struct GitInputScheme : InputScheme bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl); // FIXME: check whether rev is an ancestor of ref. From 117baee1b78f662291e980075668720179c1c455 Mon Sep 17 00:00:00 2001 From: Manuel <2084639+tennox@users.noreply.github.com> Date: Sun, 26 Jun 2022 18:00:34 +0100 Subject: [PATCH 053/119] Update src/nix/flake.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/NixOS/nix/pull/6699#discussion_r904096906 Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/nix/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 24255c247..fdb373f24 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -760,7 +760,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand if (pathExists(to2)) { auto contents2 = readFile(to2); if (contents != contents2) { - printError("refusing to overwrite existing file '%s'\n-> merge manually with '%s'", to2, from2); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); success = false; conflictedFiles.push_back(to2); } else { From 58cbbdc5e78b952bfaf8ff36e9c94ccbd08469b7 Mon Sep 17 00:00:00 2001 From: Manuel <2084639+tennox@users.noreply.github.com> Date: Sun, 26 Jun 2022 18:00:57 +0100 Subject: [PATCH 054/119] Update src/nix/flake.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/NixOS/nix/pull/6699#discussion_r904097147 Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/nix/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index fdb373f24..10bbcaf43 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -774,7 +774,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto target = readLink(from2); if (pathExists(to2)) { if (readLink(to2) != target) { - printError("refusing to overwrite existing file '%s' - please merge manually with '%s'", to2, from2); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); success = false; conflictedFiles.push_back(to2); } else { From 4374e3ec67a6c3ed8342908a5229437424926bf5 Mon Sep 17 00:00:00 2001 From: "Manu [tennox]" <2084639+tennox@users.noreply.github.com> Date: Sun, 26 Jun 2022 18:12:30 +0100 Subject: [PATCH 055/119] #6699 flake init: Apply suggestions of @thufschmitt --- src/nix/flake.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 10bbcaf43..1140548e7 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -742,7 +742,6 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand std::vector changedFiles; std::vector conflictedFiles; - auto success = false; std::function copyDir; copyDir = [&](const Path & from, const Path & to) @@ -761,7 +760,6 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto contents2 = readFile(to2); if (contents != contents2) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); - success = false; conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); @@ -775,7 +773,6 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand if (pathExists(to2)) { if (readLink(to2) != target) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); - success = false; conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); @@ -803,8 +800,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand notice(renderMarkdownToTerminal(welcomeText->getString())); } - if (!success) - throw Error("Encountered %d conflicts - please merge manually", conflictedFiles.size()); + if (!conflictedFiles.empty()) + throw Error("Encountered %d conflicts - see above", conflictedFiles.size()); } }; From ae4c9ef8e284eabf3624d9e9ad0f0b432e06da41 Mon Sep 17 00:00:00 2001 From: "Manu [tennox]" <2084639+tennox@users.noreply.github.com> Date: Sun, 26 Jun 2022 21:29:45 +0100 Subject: [PATCH 056/119] #6699 flake init: fix trying to add unchanged file After skipping because of being of identical content it tried to git add it. --- src/nix/flake.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1140548e7..895a7de76 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -777,6 +777,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } else { notice("skipping identical file: %s", from2); } + continue; } else createSymlink(target, to2); } @@ -789,7 +790,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand copyDir(templateDir, flakeDir); - if (pathExists(flakeDir + "/.git")) { + if (!changedFiles.empty() && pathExists(flakeDir + "/.git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; for (auto & s : changedFiles) args.push_back(s); runProgram("git", true, args); From cd361b31faf5a00ccd57eedf3dbd344d9ffb2faf Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 28 Jun 2022 22:43:37 -0400 Subject: [PATCH 057/119] doc: Fix typo --- src/nix/registry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/registry.md b/src/nix/registry.md index d5c9ef442..bd3575d1b 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -29,7 +29,7 @@ highest precedence: can be specified using the NixOS option `nix.registry`. * The user registry `~/.config/nix/registry.json`. This registry can - be modified by commands such as `nix flake pin`. + be modified by commands such as `nix registry pin`. * Overrides specified on the command line using the option `--override-flake`. From 455177cbe0aa4c71040857c9b3fa3ccc6312830a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jun 2022 11:29:36 +0200 Subject: [PATCH 058/119] src/libexpr/tests/primops.cc: Quote Nix expressions Otherwise they don't survive reformatting, see the failure in https://github.com/NixOS/nix/pull/6721. --- src/libexpr/tests/primops.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index f65b6593d..16cf66d2c 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -540,22 +540,22 @@ namespace nix { ASSERT_THAT(v, IsStringEq(output)); } -#define CASE(input, output) (std::make_tuple(std::string_view("builtins.toString " #input), std::string_view(output))) +#define CASE(input, output) (std::make_tuple(std::string_view("builtins.toString " input), std::string_view(output))) INSTANTIATE_TEST_SUITE_P( toString, ToStringPrimOpTest, testing::Values( - CASE("foo", "foo"), - CASE(1, "1"), - CASE([1 2 3], "1 2 3"), - CASE(.123, "0.123000"), - CASE(true, "1"), - CASE(false, ""), - CASE(null, ""), - CASE({ v = "bar"; __toString = self: self.v; }, "bar"), - CASE({ v = "bar"; __toString = self: self.v; outPath = "foo"; }, "bar"), - CASE({ outPath = "foo"; }, "foo"), - CASE(./test, "/test") + CASE(R"("foo")", "foo"), + CASE(R"(1)", "1"), + CASE(R"([1 2 3])", "1 2 3"), + CASE(R"(.123)", "0.123000"), + CASE(R"(true)", "1"), + CASE(R"(false)", ""), + CASE(R"(null)", ""), + CASE(R"({ v = "bar"; __toString = self: self.v; })", "bar"), + CASE(R"({ v = "bar"; __toString = self: self.v; outPath = "foo"; })", "bar"), + CASE(R"({ outPath = "foo"; })", "foo"), + CASE(R"(./test)", "/test") ) ); #undef CASE From 6cab5284614991ea3622492eacdceb3caf52ccff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jun 2022 12:16:51 +0200 Subject: [PATCH 059/119] Don't fail if we can't create ~/.local/share/nix/root https://hydra.nixos.org/build/182135943 --- src/libstore/store-api.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 53b1a8777..05353bce2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1326,9 +1326,14 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ auto chrootStore = getDataDir() + "/nix/root"; - if (!pathExists(chrootStore)) + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (Error & e) { + return std::make_shared(params); + } warn("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); - else + } else debug("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); Store::Params params2; params2["root"] = chrootStore; From 83f96e61a43b77677e14cdf415f1a30d37b17f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 29 Jun 2022 16:28:46 +0200 Subject: [PATCH 060/119] Add some test for `nix flake init` with conflicts --- tests/flakes.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 36bffcf3b..35cf4d8e7 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -408,8 +408,10 @@ cat > $templatesDir/trivial/flake.nix < $templatesDir/trivial/a +echo b > $templatesDir/trivial/b -git -C $templatesDir add flake.nix trivial/flake.nix +git -C $templatesDir add flake.nix trivial/ git -C $templatesDir commit -m 'Initial' nix flake check templates @@ -424,6 +426,18 @@ nix flake show $flake7Dir nix flake show $flake7Dir --json | jq git -C $flake7Dir commit -a -m 'Initial' +# Test 'nix flake init' with benign conflicts +rm -rf $flake7Dir && mkdir $flake7Dir && git -C $flake7Dir init +echo a > $flake7Dir/a +(cd $flake7Dir && nix flake init) # check idempotence + +# Test 'nix flake init' with conflicts +rm -rf $flake7Dir && mkdir $flake7Dir && git -C $flake7Dir init +echo b > $flake7Dir/a +pushd $flake7Dir +(! nix flake init) |& grep "refusing to overwrite existing file '$flake7Dir/a'" +popd + # Test 'nix flake new'. rm -rf $flake6Dir nix flake new -t templates#trivial $flake6Dir From d31c520f40101326179fe27bad7ccc2045c73156 Mon Sep 17 00:00:00 2001 From: naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Fri, 1 Jul 2022 00:29:30 +0000 Subject: [PATCH 061/119] chore: Set permissions for GitHub actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restrict the GitHub token permissions only to the required ones; this way, even if the attackers will succeed in compromising your workflow, they won’t be able to do much. - Included permissions for the action. https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) Signed-off-by: naveen <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/ci.yml | 2 ++ .github/workflows/hydra_status.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc6531ea5..1a317f267 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,8 @@ jobs: - run: nix --experimental-features 'nix-command flakes' flake check -L check_cachix: + permissions: + contents: none name: Cachix secret present for installer tests runs-on: ubuntu-latest outputs: diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml index 53e69cb2d..d85999256 100644 --- a/.github/workflows/hydra_status.yml +++ b/.github/workflows/hydra_status.yml @@ -3,6 +3,9 @@ on: schedule: - cron: "12,42 * * * *" workflow_dispatch: +permissions: + contents: read + jobs: check_hydra_status: name: Check Hydra status From 07416a6005fe035baa8646c6827904afb5226f95 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 28 Jun 2022 16:38:19 -0400 Subject: [PATCH 062/119] Allow specification of extra packages, maxLayers in Docker image --- docker.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker.nix b/docker.nix index 0cd64856f..ddf6feff5 100644 --- a/docker.nix +++ b/docker.nix @@ -4,6 +4,8 @@ , tag ? "latest" , channelName ? "nixpkgs" , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" +, extraPkgs ? [] +, maxLayers ? 100 }: let defaultPkgs = with pkgs; [ @@ -23,7 +25,7 @@ let iana-etc git openssh - ]; + ] ++ extraPkgs; users = { @@ -229,7 +231,7 @@ let in pkgs.dockerTools.buildLayeredImageWithNixDb { - inherit name tag; + inherit name tag maxLayers; contents = [ baseSystem ]; From 70083218b3727bd1279026cbe4d4179d0c7182c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 5 Jul 2022 12:06:58 +0200 Subject: [PATCH 063/119] Restrict the permissions of the CI --- .github/workflows/ci.yml | 2 ++ .github/workflows/hydra_status.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a317f267..956f81684 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ on: pull_request: push: +permissions: read-all + jobs: tests: diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml index d85999256..7766e4f8e 100644 --- a/.github/workflows/hydra_status.yml +++ b/.github/workflows/hydra_status.yml @@ -1,4 +1,7 @@ name: Hydra status + +permissions: read-all + on: schedule: - cron: "12,42 * * * *" From 541e10496a242d7c32b4e3f3ce9b4caacc04eb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 5 Jul 2022 16:28:39 +0200 Subject: [PATCH 064/119] Fix the hydra_status CI job --- .github/workflows/hydra_status.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml index 7766e4f8e..38a9c0877 100644 --- a/.github/workflows/hydra_status.yml +++ b/.github/workflows/hydra_status.yml @@ -6,8 +6,6 @@ on: schedule: - cron: "12,42 * * * *" workflow_dispatch: -permissions: - contents: read jobs: check_hydra_status: From ba1fe85b65e4e6408971bb36c40e0aad684cfc74 Mon Sep 17 00:00:00 2001 From: Gytis Ivaskevicius Date: Mon, 13 Dec 2021 09:24:24 +0200 Subject: [PATCH 065/119] Add builtins.traceVerbose Co-Authored-By: Silvan Mosberger Add builtins.traceVerbose tests --- src/libexpr/eval.hh | 3 +++ src/libexpr/primops.cc | 21 +++++++++++++++++++++ tests/lang.sh | 2 ++ 3 files changed, 26 insertions(+) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4eaa3c9b0..7db954bf4 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -646,6 +646,9 @@ struct EvalSettings : Config Setting useEvalCache{this, true, "eval-cache", "Whether to use the flake evaluation cache."}; + + Setting traceVerbose{this, false, "trace-verbose", + "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index eea274301..ac84e26c3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -970,6 +970,15 @@ static RegisterPrimOp primop_trace({ }); +/* Takes two arguments and evaluates to the second one. Used as the + * builtins.traceVerbose implementation when --trace-verbose is not enabled + */ +static void prim_second(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[1], pos); + v = *args[1]; +} + /************************************************************* * Derivations *************************************************************/ @@ -3926,6 +3935,18 @@ void EvalState::createBaseEnv() addPrimOp("__exec", 1, prim_exec); } + addPrimOp({ + .fun = evalSettings.traceVerbose ? prim_trace : prim_second, + .arity = 2, + .name = symbols.create("__traceVerbose"), + .args = { "e1", "e2" }, + .doc = R"( + Evaluate *e1* and print its abstract syntax representation on standard + error if `--trace-verbose` is enabled. Then return *e2*. This function + is useful for debugging. + )", + }); + /* Add a value containing the current Nix expression search path. */ mkList(v, searchPath.size()); int n = 0; diff --git a/tests/lang.sh b/tests/lang.sh index f09eaeb31..c0b0fc58c 100644 --- a/tests/lang.sh +++ b/tests/lang.sh @@ -5,6 +5,8 @@ export NIX_REMOTE=dummy:// nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 +nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grep -q Hello +(! nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grep -q Hello) (! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello) nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello From b2703c73a4e28a2456a599a122cc2b4ab0d33430 Mon Sep 17 00:00:00 2001 From: Gytis Ivaskevicius Date: Tue, 5 Jul 2022 19:56:39 +0300 Subject: [PATCH 066/119] builtins.traceVerbose: Post rebase fixes --- src/libexpr/primops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ac84e26c3..5fda9af75 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -973,7 +973,7 @@ static RegisterPrimOp primop_trace({ /* Takes two arguments and evaluates to the second one. Used as the * builtins.traceVerbose implementation when --trace-verbose is not enabled */ -static void prim_second(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[1], pos); v = *args[1]; @@ -3938,7 +3938,7 @@ void EvalState::createBaseEnv() addPrimOp({ .fun = evalSettings.traceVerbose ? prim_trace : prim_second, .arity = 2, - .name = symbols.create("__traceVerbose"), + .name = "__traceVerbose", .args = { "e1", "e2" }, .doc = R"( Evaluate *e1* and print its abstract syntax representation on standard From 6fa95c35c7f9d88ad7685614d275d06373394c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Jul 2022 06:46:54 +0200 Subject: [PATCH 067/119] Add a release notes entry for #4914 cc @gytis-ivaskevicius --- doc/manual/src/release-notes/rl-next.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 9684a70d4..f6a45184f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -9,3 +9,6 @@ `nix repl --file ''` or `nix repl --expr 'import {}'` This is currently guarded by the 'repl-flake' experimental feature + +* A new primop `builtins.traceVerbose` is available. It is similar to `builtins.trace` + if the `trace-verbose` setting is set to true, and it is a no-op otherwise. From eac211a171fa6253b5ae6fdcf56fc5d0aced0201 Mon Sep 17 00:00:00 2001 From: laalsaas Date: Sun, 10 Jul 2022 12:09:44 +0200 Subject: [PATCH 068/119] Fix logical and typographical errors in nix-env man page --- doc/manual/src/command-ref/nix-env.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index 8d6abaf52..a372c5eae 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -31,7 +31,7 @@ subcommand to be performed. These are documented below. Several commands, such as `nix-env -q` and `nix-env -i`, take a list of arguments that specify the packages on which to operate. These are extended regular expressions that must match the entire name of the -package. (For details on regular expressions, see regex7.) The match is +package. (For details on regular expressions, see **regex**(7).) The match is case-sensitive. The regular expression can optionally be followed by a dash and a version number; if omitted, any version of the package will match. Here are some examples: @@ -412,7 +412,7 @@ The upgrade operation determines whether a derivation `y` is an upgrade of a derivation `x` by looking at their respective `name` attributes. The names (e.g., `gcc-3.3.1` are split into two parts: the package name (`gcc`), and the version (`3.3.1`). The version part starts after the -first dash not followed by a letter. `x` is considered an upgrade of `y` +first dash not followed by a letter. `y` is considered an upgrade of `x` if their package names match, and the version of `y` is higher than that of `x`. From 5022a713051c67d503856dab4cb088827d9f222e Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Sun, 10 Jul 2022 16:50:49 -0700 Subject: [PATCH 069/119] update stale bot per RFC 0124 --- .github/stale.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index fe24942f4..ee831135a 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,10 +1,9 @@ # Configuration for probot-stale - https://github.com/probot/stale daysUntilStale: 180 -daysUntilClose: 365 +daysUntilClose: false exemptLabels: - "critical" + - "never-stale" staleLabel: "stale" -markComment: | - I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) -closeComment: | - I closed this issue due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) +markComment: false +closeComment: false From 517ce38dadc2c8d6282b6865ec0c620b7e26df3a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jul 2022 11:17:19 +0200 Subject: [PATCH 070/119] Update release notes --- doc/manual/src/release-notes/rl-next.md | 33 +++++++++++++++++++------ 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index f6a45184f..bbaa68d4f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,14 +1,31 @@ # Release X.Y (202?-??-??) -* Nix can now be built with LTO by passing `--enable-lto` to `configure`. - LTO is currently only supported when building with GCC. - * `nix repl` now takes installables on the command line, unifying the usage with other commands that use `--file` and `--expr`. Primary breaking change is for the common usage of `nix repl ''` which can be recovered with - `nix repl --file ''` or `nix repl --expr 'import {}'` - - This is currently guarded by the 'repl-flake' experimental feature + `nix repl --file ''` or `nix repl --expr 'import {}'`. -* A new primop `builtins.traceVerbose` is available. It is similar to `builtins.trace` - if the `trace-verbose` setting is set to true, and it is a no-op otherwise. + This is currently guarded by the `repl-flake` experimental feature. + +* A new function `builtins.traceVerbose` is available. It is similar + to `builtins.trace` if the `trace-verbose` setting is set to true, + and it is a no-op otherwise. + +* `nix search` has a new flag `--exclude` to filter out packages. + +* On Linux, if `/nix` doesn't exist and cannot be created and you're + not running as root, Nix will automatically use + `~/.local/share/nix/root` as a chroot store. This enables non-root + users to download the statically linked Nix binary and have it work + out of the box, e.g. + + ``` + # ~/nix run nixpkgs#hello + warning: '/nix' does not exists, so Nix will use '/home/ubuntu/.local/share/nix/root' as a chroot store + Hello, world! + ``` + +* `flake-registry.json` is now fetched from `channels.nixos.org`. + +* Nix can now be built with LTO by passing `--enable-lto` to `configure`. + LTO is currently only supported when building with GCC. From 711b2e1f48316d80853635408c518e3562a1fa37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 20 Jun 2022 04:15:38 +0200 Subject: [PATCH 071/119] Fix flake input completion for `InstallablesCommand`s Defers completion of flake inputs until the whole command line is parsed so that we know what flakes we need to complete the inputs of. Previously, `nix build flake --update-input ` always behaved like `nix build . --update-input `. --- src/libcmd/command.hh | 4 ++++ src/libcmd/installables.cc | 34 ++++++++++++++++++++-------------- src/libutil/args.cc | 10 +++++++++- src/libutil/args.hh | 7 +++++++ src/nix/main.cc | 5 ++++- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index cab379b84..ffa8e784f 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -77,12 +77,16 @@ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; + std::optional needsFlakeInputCompletion = {}; + MixFlakeOptions(); virtual std::vector getFlakesForCompletion() { return {}; } void completeFlakeInput(std::string_view prefix); + + void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1bcef4172..81c8dd062 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -23,18 +23,6 @@ namespace nix { -void MixFlakeOptions::completeFlakeInput(std::string_view prefix) -{ - auto evalState = getEvalState(); - for (auto & flakeRefS : getFlakesForCompletion()) { - auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); - } -} - MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; @@ -87,7 +75,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, .completer = {[&](size_t, std::string_view prefix) { - completeFlakeInput(prefix); + needsFlakeInputCompletion = {std::string(prefix)}; }} }); @@ -104,7 +92,7 @@ MixFlakeOptions::MixFlakeOptions() }}, .completer = {[&](size_t n, std::string_view prefix) { if (n == 0) - completeFlakeInput(prefix); + needsFlakeInputCompletion = {std::string(prefix)}; else if (n == 1) completeFlakeRef(getEvalState()->store, prefix); }} @@ -137,6 +125,24 @@ MixFlakeOptions::MixFlakeOptions() }); } +void MixFlakeOptions::completeFlakeInput(std::string_view prefix) +{ + auto evalState = getEvalState(); + for (auto & flakeRefS : getFlakesForCompletion()) { + auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions->add(input.first); + } +} + +void MixFlakeOptions::completionHook() +{ + if (auto & prefix = needsFlakeInputCompletion) + completeFlakeInput(*prefix); +} + SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) { addFlag({ diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 4b8c55686..44b63f0f6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -124,7 +124,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { - if (flag.handler.arity == ArityAny) break; + if (flag.handler.arity == ArityAny || anyCompleted) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } if (auto prefix = needsCompletion(*pos)) { @@ -362,6 +362,14 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } +void MultiCommand::completionHook() +{ + if (command) + return command->second->completionHook(); + else + return Args::completionHook(); +} + nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 07c017719..84866f12b 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -148,6 +148,11 @@ protected: argument (if any) have been processed. */ virtual void initialFlagsProcessed() {} + /* Called after the command line has been processed if we need to generate + completions. Useful for commands that need to know the whole command line + in order to know what completions to generate. */ + virtual void completionHook() { } + public: void addFlag(Flag && flag); @@ -223,6 +228,8 @@ public: bool processArgs(const Strings & args, bool finish) override; + void completionHook() override; + nlohmann::json toJSON() override; }; diff --git a/src/nix/main.cc b/src/nix/main.cc index f398e3118..f6138cbe6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -342,7 +342,10 @@ void mainWrapped(int argc, char * * argv) if (!completions) throw; } - if (completions) return; + if (completions) { + args.completionHook(); + return; + } if (args.showVersion) { printVersion(programName); From 69ea265fd26e6b503bb52566ce6f5f12e0a75661 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 Jul 2022 10:21:12 -0600 Subject: [PATCH 072/119] 'tryEval' not 'try clause' --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 28256ec5c..956c4b474 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -838,7 +838,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & printError("%s\n\n", error->what()); if (trylevel > 0 && error->info().level != lvlInfo) - printError("This exception occurred in a try clause. use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); + printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL); } From 6ac8200ff5d21d7c4464b4b3a2d3716fa4b942fd Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 Jul 2022 10:21:40 -0600 Subject: [PATCH 073/119] use util.hh class instead of local --- src/libexpr/primops.cc | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ecc1c136a..3a07e43a7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -846,15 +846,6 @@ static RegisterPrimOp primop_floor({ .fun = prim_floor, }); -class Counter -{ - private: - int &counter; - public: - Counter(int &counter) :counter(counter) { counter++; } - ~Counter() { counter--; } -}; - /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -862,7 +853,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va auto attrs = state.buildBindings(2); /* increment state.trylevel, and decrement it when this function returns. */ - Counter trylevel(state.trylevel); + MaintainCount trylevel(state.trylevel); void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; if (state.debugRepl && state.ignoreTry) From a3629ab0ccd40a4492ac99424d84b3649df8b057 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 Jul 2022 10:47:09 -0600 Subject: [PATCH 074/119] move ignore-try to EvalSettings --- src/libcmd/command.cc | 9 --------- src/libexpr/eval.cc | 1 - src/libexpr/eval.hh | 8 +++++++- src/libexpr/primops.cc | 2 +- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 940fd5b23..14bb27936 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -91,12 +91,6 @@ EvalCommand::EvalCommand() .description = "start an interactive environment if evaluation fails", .handler = {&startReplOnEvalErrors, true}, }); - - addFlag({ - .longName = "ignore-try", - .description = "ignore exceptions in try clauses during debug", - .handler = {&ignoreExceptionsDuringTry, true}, - }); } EvalCommand::~EvalCommand() @@ -128,9 +122,6 @@ ref EvalCommand::getEvalState() if (startReplOnEvalErrors) { evalState->debugRepl = &runRepl; }; - if (ignoreExceptionsDuringTry) { - evalState->ignoreTry = ignoreExceptionsDuringTry; - }; } return ref(evalState); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 956c4b474..f485e2fed 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -467,7 +467,6 @@ EvalState::EvalState( , debugRepl(nullptr) , debugStop(false) , debugQuit(false) - , ignoreTry(false) , trylevel(0) , regexCache(makeRegexCache()) #if HAVE_BOEHMGC diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9aff77042..b8903c06c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -130,7 +130,6 @@ public: void (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; bool debugQuit; - bool ignoreTry; int trylevel; std::list debugTraces; std::map> exprEnvs; @@ -648,6 +647,13 @@ struct EvalSettings : Config Setting useEvalCache{this, true, "eval-cache", "Whether to use the flake evaluation cache."}; + + Setting ignoreExceptionsDuringTry{this, false, "ignore-try", + R"( + If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in + debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. + )"}; + }; extern EvalSettings evalSettings; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3a07e43a7..2201ca0c4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -856,7 +856,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va MaintainCount trylevel(state.trylevel); void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl && state.ignoreTry) + if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) { /* to prevent starting the repl from exceptions withing a tryEval, null it. */ savedDebugRepl = state.debugRepl; From c364e0b098e1abc5bb06e0e86b3525c7bef33e15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jul 2022 20:56:19 +0200 Subject: [PATCH 075/119] Fix buildStatic.aarch64-linux Commit 925b97522497e9c0f7a385c904410e560796208f accidentally enabled use of the gold linker on aarch64-linux, which apparently doesn't work. https://hydra.nixos.org/build/183500621 --- flake.nix | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 22367df71..1ee040441 100644 --- a/flake.nix +++ b/flake.nix @@ -54,7 +54,7 @@ # we want most of the time and for backwards compatibility forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages); - commonDeps = pkgs: with pkgs; rec { + commonDeps = { pkgs, isStatic ? false }: with pkgs; rec { # Use "busybox-sandbox-shell" if present, # if not (legacy) fallback and hope it's sufficient. sh = pkgs.busybox-sandbox-shell or (busybox.override { @@ -85,6 +85,8 @@ lib.optionals stdenv.isLinux [ "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" + ] + ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ "LDFLAGS=-fuse-ld=gold" ]; @@ -170,7 +172,7 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; - testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation { + testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; name = @@ -281,7 +283,7 @@ # Forward from the previous stage as we don’t want it to pick the lowdown override nixUnstable = prev.nixUnstable; - nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation { + nix = with final; with commonDeps { inherit pkgs; }; currentStdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -448,7 +450,7 @@ # Line coverage analysis. coverage = with nixpkgsFor.x86_64-linux; - with commonDeps pkgs; + with commonDeps { inherit pkgs; }; releaseTools.coverageAnalysis { name = "nix-coverage-${version}"; @@ -559,7 +561,7 @@ } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { nix-static = let nixpkgs = nixpkgsFor.${system}.pkgsStatic; - in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation { + in with commonDeps { pkgs = nixpkgs; isStatic = true; }; nixpkgs.stdenv.mkDerivation { name = "nix-${version}"; src = self; @@ -630,7 +632,7 @@ inherit system crossSystem; overlays = [ self.overlays.default ]; }; - in with commonDeps nixpkgsCross; nixpkgsCross.stdenv.mkDerivation { + in with commonDeps { pkgs = nixpkgsCross; }; nixpkgsCross.stdenv.mkDerivation { name = "nix-${version}"; src = self; @@ -673,7 +675,7 @@ devShells = forAllSystems (system: forAllStdenvs (stdenv: with nixpkgsFor.${system}; - with commonDeps pkgs; + with commonDeps { inherit pkgs; }; nixpkgsFor.${system}.${stdenv}.mkDerivation { name = "nix"; From 28e913c605d36711546706b7ddbdb8ad711a2486 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jul 2022 21:10:23 +0200 Subject: [PATCH 076/119] Branch 2.10 release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.10.md | 31 +++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 29 ----------------------- 3 files changed, 32 insertions(+), 29 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.10.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 825a8b4c0..9728728aa 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -72,6 +72,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md) diff --git a/doc/manual/src/release-notes/rl-2.10.md b/doc/manual/src/release-notes/rl-2.10.md new file mode 100644 index 000000000..b99dbeef0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.10.md @@ -0,0 +1,31 @@ +# Release 2.10 (2022-07-11) + +* `nix repl` now takes installables on the command line, unifying the usage + with other commands that use `--file` and `--expr`. Primary breaking change + is for the common usage of `nix repl ''` which can be recovered with + `nix repl --file ''` or `nix repl --expr 'import {}'`. + + This is currently guarded by the `repl-flake` experimental feature. + +* A new function `builtins.traceVerbose` is available. It is similar + to `builtins.trace` if the `trace-verbose` setting is set to true, + and it is a no-op otherwise. + +* `nix search` has a new flag `--exclude` to filter out packages. + +* On Linux, if `/nix` doesn't exist and cannot be created and you're + not running as root, Nix will automatically use + `~/.local/share/nix/root` as a chroot store. This enables non-root + users to download the statically linked Nix binary and have it work + out of the box, e.g. + + ``` + # ~/nix run nixpkgs#hello + warning: '/nix' does not exists, so Nix will use '/home/ubuntu/.local/share/nix/root' as a chroot store + Hello, world! + ``` + +* `flake-registry.json` is now fetched from `channels.nixos.org`. + +* Nix can now be built with LTO by passing `--enable-lto` to `configure`. + LTO is currently only supported when building with GCC. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index bbaa68d4f..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,31 +1,2 @@ # Release X.Y (202?-??-??) -* `nix repl` now takes installables on the command line, unifying the usage - with other commands that use `--file` and `--expr`. Primary breaking change - is for the common usage of `nix repl ''` which can be recovered with - `nix repl --file ''` or `nix repl --expr 'import {}'`. - - This is currently guarded by the `repl-flake` experimental feature. - -* A new function `builtins.traceVerbose` is available. It is similar - to `builtins.trace` if the `trace-verbose` setting is set to true, - and it is a no-op otherwise. - -* `nix search` has a new flag `--exclude` to filter out packages. - -* On Linux, if `/nix` doesn't exist and cannot be created and you're - not running as root, Nix will automatically use - `~/.local/share/nix/root` as a chroot store. This enables non-root - users to download the statically linked Nix binary and have it work - out of the box, e.g. - - ``` - # ~/nix run nixpkgs#hello - warning: '/nix' does not exists, so Nix will use '/home/ubuntu/.local/share/nix/root' as a chroot store - Hello, world! - ``` - -* `flake-registry.json` is now fetched from `channels.nixos.org`. - -* Nix can now be built with LTO by passing `--enable-lto` to `configure`. - LTO is currently only supported when building with GCC. From 32effccb51783a36ce88607a7a404083e84ab3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 11 Jul 2022 15:16:19 +0200 Subject: [PATCH 077/119] Add some tests for the CLI completion --- tests/completions.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 2 files changed, 49 insertions(+) create mode 100644 tests/completions.sh diff --git a/tests/completions.sh b/tests/completions.sh new file mode 100644 index 000000000..4d6beccf8 --- /dev/null +++ b/tests/completions.sh @@ -0,0 +1,48 @@ +source common.sh + +cd "$TEST_ROOT" + +mkdir -p dep && pushd dep +cat < flake.nix +{ + outputs = i: { }; +} +EOF +popd +mkdir -p foo && pushd foo +cat < flake.nix +{ + inputs.a.url = "path:$(realpath ../dep)"; + + outputs = i: { + sampleOutput = 1; + }; +} +EOF + +popd + +# Test the completion of a subcommand +[[ $(printf "normal\nbuild\t\n") == $(NIX_GET_COMPLETIONS=1 nix buil) ]] +[[ $(printf "normal\nmetadata\t\n") == $(NIX_GET_COMPLETIONS=2 nix flake metad) ]] + +# Filename completion +[[ $(printf "filenames\n./foo\t\n") == $(NIX_GET_COMPLETIONS=2 nix build ./f) ]] +[[ $(printf "filenames\n") == $(NIX_GET_COMPLETIONS=2 nix build ./nonexistent) ]] + +# Input override completion +[[ $(printf "normal\na\t\n") == $(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '') ]] + +# Cli flag completion +NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" + +# Config option completion +## With `--option` +NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-import-from-derivation" +## As a cli flag – not working atm +# NIX_GET_COMPLETIONS=2 nix build --allow-import-from | grep -- "allow-import-from-derivation" + + +# Attr path completions +[[ $(printf "attrs\n./foo#sampleOutput\t\n") == $(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam) ]] +[[ $(printf "attrs\noutputs\t\n") == $(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp) ]] diff --git a/tests/local.mk b/tests/local.mk index ae15c70f9..e0579f503 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -102,6 +102,7 @@ nix_tests = \ suggestions.sh \ store-ping.sh \ fetchClosure.sh \ + completion.sh \ impure-derivations.sh ifeq ($(HAVE_LIBCPUID), 1) From 260fb837de849a10571f19f81adbd6091a28c815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Jul 2022 09:19:05 +0200 Subject: [PATCH 078/119] Fix the name of the completions test --- tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local.mk b/tests/local.mk index e0579f503..82e47729b 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -102,7 +102,7 @@ nix_tests = \ suggestions.sh \ store-ping.sh \ fetchClosure.sh \ - completion.sh \ + completions.sh \ impure-derivations.sh ifeq ($(HAVE_LIBCPUID), 1) From 07e14d3ef09e7852fce3a3850cd50eea032e3753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Jul 2022 09:19:15 +0200 Subject: [PATCH 079/119] Harden the comparisons in the completion test - Don't use `printf` for the expected result, but just use bash's `$' '` litteral strings - Quote the `nix` call result - Invert the order in the comparisons (just because it feels more natural) --- tests/completions.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/completions.sh b/tests/completions.sh index 4d6beccf8..b7bb31aea 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -23,15 +23,15 @@ EOF popd # Test the completion of a subcommand -[[ $(printf "normal\nbuild\t\n") == $(NIX_GET_COMPLETIONS=1 nix buil) ]] -[[ $(printf "normal\nmetadata\t\n") == $(NIX_GET_COMPLETIONS=2 nix flake metad) ]] +[[ "$(NIX_GET_COMPLETIONS=1 nix buil)" == $'normal\nbuild\t' ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix flake metad)" == $'normal\nmetadata\t' ]] # Filename completion -[[ $(printf "filenames\n./foo\t\n") == $(NIX_GET_COMPLETIONS=2 nix build ./f) ]] -[[ $(printf "filenames\n") == $(NIX_GET_COMPLETIONS=2 nix build ./nonexistent) ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix build ./f)" == $'filenames\n./foo\t' ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix build ./nonexistent)" == $'filenames' ]] # Input override completion -[[ $(printf "normal\na\t\n") == $(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '') ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" @@ -44,5 +44,5 @@ NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-impo # Attr path completions -[[ $(printf "attrs\n./foo#sampleOutput\t\n") == $(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam) ]] -[[ $(printf "attrs\noutputs\t\n") == $(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp) ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam)" == $'attrs\n./foo#sampleOutput\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp)" == $'attrs\noutputs\t' ]] From 21c443d4fd0dc4e28f4af085aef711d5ce30c5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Jul 2022 09:37:57 +0200 Subject: [PATCH 080/119] Test the tilde expansion for the flake completion Also add a disabled test for when the `--override-input` flag comes *before* the flake ref --- tests/completions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/completions.sh b/tests/completions.sh index b7bb31aea..a510e28b0 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -32,6 +32,10 @@ popd # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] +## With tilde expansion +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +## Out of order – not working atm. Should have been fixed by #6693 but apparently not +# [[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' ./foo)" == $'normal\na\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" From c1c37f32002e2e2ad167bcac2e5d76a3a3ad6a35 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 13 Jun 2022 20:39:09 +0200 Subject: [PATCH 081/119] flakes: throw an error if `follows`-declaration for an input is invalid I recently got fairly confused why the following expression didn't have any effect { description = "Foobar"; inputs.sops-nix = { url = github:mic92/sops-nix; inputs.nixpkgs_22_05.follows = "nixpkgs"; }; } until I found out that the input was called `nixpkgs-22_05` (please note the dash vs. underscore). IMHO it's not a good idea to not throw an error in that case and probably leave end-users rather confused, so I implemented a small check for that which basically checks whether `follows`-declaration from overrides actually have corresponding inputs in the transitive flake. In fact this was done by accident already in our own test-suite where the removal of a `follows` was apparently forgotten[1]. Since the key of the `std::map` that holds the `overrides` is a vector and we have to find the last element of each vector (i.e. the override) this has to be done with a for loop in O(n) complexity with `n` being the total amount of overrides (which shouldn't be that large though). Please note that this doesn't work with nested expressions, i.e. inputs.fenix.inputs.nixpkgs.follows = "..."; which is a known problem[2]. For the expression demonstrated above, an error like this will be thrown: error: sops-nix has a `follows'-declaration for a non-existant input nixpkgs_22_05! [1] 2664a216e57169ec57d7f51be1b8383c1be83fd5 [2] https://github.com/NixOS/nix/issues/5790 --- src/libexpr/flake/flake.cc | 23 +++++++++++++++++++++++ tests/flakes.sh | 21 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 920726b73..b97780a9c 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -373,6 +373,29 @@ LockedFlake lockFlake( { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + auto overrides2 = overrides; + for (auto & [inputPath, inputOverride] : overrides2) { + auto inputPath2(inputPath); + auto follow = inputPath2.back(); + inputPath2.pop_back(); + if (inputPath2 == inputPathPrefix + && flakeInputs.find(follow) == flakeInputs.end() + ) { + std::string root; + for (auto & element : inputPath2) { + root.append(element); + if (element != inputPath2.back()) { + root.append(".inputs."); + } + } + throw Error( + "%s has a `follows'-declaration for a non-existant input %s!", + root, + follow + ); + } + } + /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ for (auto & [id, input] : flakeInputs) { diff --git a/tests/flakes.sh b/tests/flakes.sh index 35cf4d8e7..868c410f4 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -801,6 +801,8 @@ nix flake metadata $flakeFollowsA nix flake update $flakeFollowsA +nix flake lock $flakeFollowsA + oldLock="$(cat "$flakeFollowsA/flake.lock")" # Ensure that locking twice doesn't change anything @@ -823,7 +825,6 @@ cat > $flakeFollowsA/flake.nix <$flakeFollowsA/flake.nix <&1 | grep "error: B has a \`follows'-declaration for a non-existant input invalid!" From 411111a3bc0e47520797106e1697aaa11631a101 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 12 Jul 2022 11:22:35 +0200 Subject: [PATCH 082/119] Turn error for non-existant follows into a warning --- src/libexpr/flake/flake.cc | 5 ++--- tests/flakes.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index b97780a9c..c59a42d56 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -373,8 +373,7 @@ LockedFlake lockFlake( { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); - auto overrides2 = overrides; - for (auto & [inputPath, inputOverride] : overrides2) { + for (auto [inputPath, inputOverride] : overrides) { auto inputPath2(inputPath); auto follow = inputPath2.back(); inputPath2.pop_back(); @@ -388,7 +387,7 @@ LockedFlake lockFlake( root.append(".inputs."); } } - throw Error( + warn( "%s has a `follows'-declaration for a non-existant input %s!", root, follow diff --git a/tests/flakes.sh b/tests/flakes.sh index 868c410f4..9087e1978 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -877,4 +877,4 @@ EOF git -C $flakeFollowsA add flake.nix -nix flake lock $flakeFollowsA 2>&1 | grep "error: B has a \`follows'-declaration for a non-existant input invalid!" +nix flake lock $flakeFollowsA 2>&1 | grep "warning: B has a \`follows'-declaration for a non-existant input invalid!" From 1f771065f1353ba462d73641b047b4fb2f02f482 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 12 Jul 2022 11:25:33 +0200 Subject: [PATCH 083/119] Move follows-check into its own function --- src/libexpr/flake/flake.cc | 47 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c59a42d56..906e11251 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -353,26 +353,15 @@ LockedFlake lockFlake( std::vector parents; std::function node, const InputPath & inputPathPrefix, - std::shared_ptr oldNode, - const InputPath & lockRootPath, - const Path & parentPath, - bool trustLock)> - computeLocks; + const FlakeInputs & flakeInputs + )> + checkFollowsDeclarations; - computeLocks = [&]( - const FlakeInputs & flakeInputs, - std::shared_ptr node, + checkFollowsDeclarations = [&]( const InputPath & inputPathPrefix, - std::shared_ptr oldNode, - const InputPath & lockRootPath, - const Path & parentPath, - bool trustLock) - { - debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); - + const FlakeInputs & flakeInputs + ) { for (auto [inputPath, inputOverride] : overrides) { auto inputPath2(inputPath); auto follow = inputPath2.back(); @@ -394,6 +383,30 @@ LockedFlake lockFlake( ); } } + }; + + std::function node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode, + const InputPath & lockRootPath, + const Path & parentPath, + bool trustLock)> + computeLocks; + + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode, + const InputPath & lockRootPath, + const Path & parentPath, + bool trustLock) + { + debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + + checkFollowsDeclarations(inputPathPrefix, flakeInputs); /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ From f6a434c8a4d76973a26e8cbd938be2beb29696de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jul 2022 11:53:24 +0200 Subject: [PATCH 084/119] Fix debug message --- src/libexpr/eval-cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index dbfd8e70b..0d83b6cfe 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -486,7 +486,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro return nullptr; else if (std::get_if(&attr->second)) { if (forceErrors) - debug("reevaluating failed cached attribute '%s'"); + debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name)); else throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); } else From c9d406ba04b81c21f187348fe52fa6e1f95ad78c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jul 2022 16:09:25 +0200 Subject: [PATCH 085/119] Fix --no-daemon installation It was accidentally triggering the auto-chroot code path because /nix/var/nix didn't exist. Fixes #6790. --- scripts/install-nix-from-closure.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index d543b4463..cd3cf6670 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -148,7 +148,9 @@ if ! [ -w "$dest" ]; then exit 1 fi -mkdir -p "$dest/store" +# The auto-chroot code in openFromNonUri() checks for the +# non-existence of /nix/var/nix, so we need to create it here. +mkdir -p "$dest/store" "$dest/var/nix" printf "copying Nix to %s..." "${dest}/store" >&2 # Insert a newline if no progress is shown. From 694a9dc282dd9f07e9f8f1bc7362bef50588389c Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Wed, 13 Jul 2022 01:10:32 +0900 Subject: [PATCH 086/119] Fix typo in flake.cc non-existant -> non-existent --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 906e11251..233e52407 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -377,7 +377,7 @@ LockedFlake lockFlake( } } warn( - "%s has a `follows'-declaration for a non-existant input %s!", + "%s has a `follows'-declaration for a non-existent input %s!", root, follow ); From d34a333e2ea6f7c3a8f86b821edd0542661d036a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Jul 2022 10:25:28 +0200 Subject: [PATCH 087/119] =?UTF-8?q?Fix=20the=20=E2=80=9Cout=20of=20order?= =?UTF-8?q?=E2=80=9D=20completion=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--override-input` id snarky because it takes two arguments, so it doesn't play well when completed in the middle of the CLI (since the argument just after gets interpreted as its second argument). So use `--update-input` instead --- tests/completions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/completions.sh b/tests/completions.sh index a510e28b0..75b567146 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -34,8 +34,8 @@ popd [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] -## Out of order – not working atm. Should have been fixed by #6693 but apparently not -# [[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' ./foo)" == $'normal\na\t' ]] +## Out of order +[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" From b052e7e71ddf85921580415039a5a86a98ce042c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Jul 2022 10:29:47 +0200 Subject: [PATCH 088/119] Add some more completion tests - Test another command than `build` - Test with two input flakes --- tests/completions.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/completions.sh b/tests/completions.sh index 75b567146..522aa1c86 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -2,25 +2,32 @@ source common.sh cd "$TEST_ROOT" -mkdir -p dep && pushd dep -cat < flake.nix +mkdir -p dep +cat < dep/flake.nix { outputs = i: { }; } EOF -popd -mkdir -p foo && pushd foo -cat < flake.nix +mkdir -p foo +cat < foo/flake.nix { - inputs.a.url = "path:$(realpath ../dep)"; + inputs.a.url = "path:$(realpath dep)"; outputs = i: { sampleOutput = 1; }; } EOF +mkdir -p bar +cat < bar/flake.nix +{ + inputs.b.url = "path:$(realpath dep)"; -popd + outputs = i: { + sampleOutput = 1; + }; +} +EOF # Test the completion of a subcommand [[ "$(NIX_GET_COMPLETIONS=1 nix buil)" == $'normal\nbuild\t' ]] @@ -32,10 +39,14 @@ popd # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +## With multiple input flakes +[[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] ## Out of order [[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" @@ -46,7 +57,6 @@ NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-impo ## As a cli flag – not working atm # NIX_GET_COMPLETIONS=2 nix build --allow-import-from | grep -- "allow-import-from-derivation" - # Attr path completions [[ "$(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam)" == $'attrs\n./foo#sampleOutput\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp)" == $'attrs\noutputs\t' ]] From 12df8885cc70499cc8fa2bfe73992c6d37ec332e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 13:39:06 +0200 Subject: [PATCH 089/119] Simplify the check for overrides on non-existent inputs --- src/libexpr/flake/flake.cc | 47 ++++++++++---------------------------- tests/flakes.sh | 4 +++- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 233e52407..cc9be1336 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -352,39 +352,6 @@ LockedFlake lockFlake( std::vector parents; - std::function - checkFollowsDeclarations; - - checkFollowsDeclarations = [&]( - const InputPath & inputPathPrefix, - const FlakeInputs & flakeInputs - ) { - for (auto [inputPath, inputOverride] : overrides) { - auto inputPath2(inputPath); - auto follow = inputPath2.back(); - inputPath2.pop_back(); - if (inputPath2 == inputPathPrefix - && flakeInputs.find(follow) == flakeInputs.end() - ) { - std::string root; - for (auto & element : inputPath2) { - root.append(element); - if (element != inputPath2.back()) { - root.append(".inputs."); - } - } - warn( - "%s has a `follows'-declaration for a non-existent input %s!", - root, - follow - ); - } - } - }; - std::function node, @@ -406,8 +373,6 @@ LockedFlake lockFlake( { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); - checkFollowsDeclarations(inputPathPrefix, flakeInputs); - /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ for (auto & [id, input] : flakeInputs) { @@ -419,6 +384,18 @@ LockedFlake lockFlake( } } + /* Check whether this input has overrides for a + non-existent input. */ + for (auto [inputPath, inputOverride] : overrides) { + auto inputPath2(inputPath); + auto follow = inputPath2.back(); + inputPath2.pop_back(); + if (inputPath2 == inputPathPrefix && !flakeInputs.count(follow)) + warn( + "input '%s' has an override for a non-existent input '%s'", + printInputPath(inputPathPrefix), follow); + } + /* Go over the flake inputs, resolve/fetch them if necessary (i.e. if they're new or the flakeref changed from what's in the lock file). */ diff --git a/tests/flakes.sh b/tests/flakes.sh index 9087e1978..9838848eb 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -869,6 +869,7 @@ cat >$flakeFollowsA/flake.nix <&1 | grep "warning: B has a \`follows'-declaration for a non-existant input invalid!" +nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" +nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" From 19190c2346b84916c64737d895b87b570c673b96 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 13:17:10 +0200 Subject: [PATCH 090/119] tests/flakes.sh: Make sure flake7 is clean Cherry-picked from the lazy-trees branch, where we no longer write a lock file if any of the inputs is dirty. --- tests/flakes.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 9087e1978..265e4a456 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -28,6 +28,15 @@ flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE +initRepo() { + local repo="$1" + local extraArgs="$2" + + git -C $repo init $extraArgs + git -C $repo config user.email "foobar@example.com" + git -C $repo config user.name "Foobar" +} + for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB $flakeFollowsA; do rm -rf $repo $repo.tmp mkdir -p $repo @@ -38,9 +47,7 @@ for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeD extraArgs="--initial-branch=main" fi - git -C $repo init $extraArgs - git -C $repo config user.email "foobar@example.com" - git -C $repo config user.name "Foobar" + initRepo "$repo" "$extraArgs" done cat > $flake1Dir/flake.nix < $flake7Dir/a (cd $flake7Dir && nix flake init) # check idempotence # Test 'nix flake init' with conflicts -rm -rf $flake7Dir && mkdir $flake7Dir && git -C $flake7Dir init +rm -rf $flake7Dir && mkdir $flake7Dir && initRepo "$flake7Dir" echo b > $flake7Dir/a pushd $flake7Dir (! nix flake init) |& grep "refusing to overwrite existing file '$flake7Dir/a'" popd +git -C $flake7Dir commit -a -m 'Changed' # Test 'nix flake new'. rm -rf $flake6Dir From 420957e149256677046b2433135f53ab0dadab3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 14:09:35 +0200 Subject: [PATCH 091/119] Move flakes tests to a subdirectory --- tests/{ => flakes}/flakes.sh | 6 +++--- tests/{flakes-run.sh => flakes/run.sh} | 4 ++-- tests/local.mk | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename tests/{ => flakes}/flakes.sh (99%) rename tests/{flakes-run.sh => flakes/run.sh} (92%) diff --git a/tests/flakes.sh b/tests/flakes/flakes.sh similarity index 99% rename from tests/flakes.sh rename to tests/flakes/flakes.sh index 6574cabf1..b14c79358 100644 --- a/tests/flakes.sh +++ b/tests/flakes/flakes.sh @@ -1,4 +1,4 @@ -source common.sh +source ../common.sh if [[ -z $(type -p git) ]]; then echo "Git not installed; skipping flake tests" @@ -66,7 +66,7 @@ cat > $flake1Dir/flake.nix < $flake3Dir/flake.nix < flake.nix diff --git a/tests/local.mk b/tests/local.mk index ae15c70f9..021aae35e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,6 +1,6 @@ nix_tests = \ - flakes.sh \ - flakes-run.sh \ + flakes/flakes.sh \ + flakes/run.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ From c591efafd3e967816964600722265b037872200a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 14:40:39 +0200 Subject: [PATCH 092/119] Split off the Mercurial flake tests --- tests/flakes/common.sh | 36 +++++++++++++++++ tests/flakes/flakes.sh | 81 ++++++--------------------------------- tests/flakes/mercurial.sh | 46 ++++++++++++++++++++++ tests/local.mk | 1 + 4 files changed, 94 insertions(+), 70 deletions(-) create mode 100644 tests/flakes/common.sh create mode 100644 tests/flakes/mercurial.sh diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh new file mode 100644 index 000000000..aa45dffa9 --- /dev/null +++ b/tests/flakes/common.sh @@ -0,0 +1,36 @@ +source ../common.sh + +registry=$TEST_ROOT/registry.json + +writeSimpleFlake() { + local flakeDir="$1" + cat > $flakeDir/flake.nix < $flakeDir/flake.nix < $flake1Dir/flake.nix < $templatesDir/flake.nix < $flake5Dir/flake.nix < Date: Wed, 13 Jul 2022 20:16:39 +0200 Subject: [PATCH 093/119] Split off the circular flake import tests --- tests/flakes/circular.sh | 49 +++++++++++++++++++++++++++++ tests/flakes/common.sh | 19 ++++++++++++ tests/flakes/flakes.sh | 66 +++------------------------------------- tests/local.mk | 1 + 4 files changed, 74 insertions(+), 61 deletions(-) create mode 100644 tests/flakes/circular.sh diff --git a/tests/flakes/circular.sh b/tests/flakes/circular.sh new file mode 100644 index 000000000..09cd02edf --- /dev/null +++ b/tests/flakes/circular.sh @@ -0,0 +1,49 @@ +# Test circular flake dependencies. +source ./common.sh + +requireGit + +flakeA=$TEST_ROOT/flakeA +flakeB=$TEST_ROOT/flakeB + +createGitRepo $flakeA +createGitRepo $flakeB + +cat > $flakeA/flake.nix < $flakeB/flake.nix < $flakeDir/flake.nix < $flake7Dir/a (cd $flake7Dir && nix flake init) # check idempotence # Test 'nix flake init' with conflicts -rm -rf $flake7Dir && mkdir $flake7Dir && initRepo "$flake7Dir" +createGitRepo "$flake7Dir" echo b > $flake7Dir/a pushd $flake7Dir (! nix flake init) |& grep "refusing to overwrite existing file '$flake7Dir/a'" @@ -641,45 +624,6 @@ nix flake lock $flake3Dir --update-input flake2/flake1 # Test 'nix flake metadata --json'. nix flake metadata $flake3Dir --json | jq . -# Test circular flake dependencies. -cat > $flakeA/flake.nix < $flakeB/flake.nix < Date: Wed, 13 Jul 2022 20:36:11 +0200 Subject: [PATCH 094/119] Split off 'nix flake init' tests --- tests/flakes/common.sh | 18 ++++++++ tests/flakes/flakes.sh | 99 ++++++------------------------------------ tests/flakes/init.sh | 87 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 4 files changed, 119 insertions(+), 86 deletions(-) create mode 100644 tests/flakes/init.sh diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index 43a589507..c333733c2 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -30,6 +30,13 @@ EOF cp ../simple.nix ../simple.builder.sh ../config.nix $flakeDir/ } +createSimpleGitFlake() { + local flakeDir="$1" + writeSimpleFlake $flakeDir + git -C $flakeDir add flake.nix simple.nix simple.builder.sh config.nix + git -C $flakeDir commit -m 'Initial' +} + writeDependentFlake() { local flakeDir="$1" cat > $flakeDir/flake.nix < $flakeDir/flake.nix < $flake2Dir/flake.nix < $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < $templatesDir/trivial/a -echo b > $templatesDir/trivial/b - -git -C $templatesDir add flake.nix trivial/ -git -C $templatesDir commit -m 'Initial' - -nix flake check templates -nix flake show templates -nix flake show templates --json | jq - -(cd $flake7Dir && nix flake init) -(cd $flake7Dir && nix flake init) # check idempotence -git -C $flake7Dir add flake.nix -nix flake check $flake7Dir -nix flake show $flake7Dir -nix flake show $flake7Dir --json | jq -git -C $flake7Dir commit -a -m 'Initial' - -# Test 'nix flake init' with benign conflicts -createGitRepo "$flake7Dir" -echo a > $flake7Dir/a -(cd $flake7Dir && nix flake init) # check idempotence - -# Test 'nix flake init' with conflicts -createGitRepo "$flake7Dir" -echo b > $flake7Dir/a -pushd $flake7Dir -(! nix flake init) |& grep "refusing to overwrite existing file '$flake7Dir/a'" -popd -git -C $flake7Dir commit -a -m 'Changed' - -# Test 'nix flake new'. -rm -rf $flake6Dir -nix flake new -t templates#trivial $flake6Dir -nix flake new -t templates#trivial $flake6Dir # check idempotence -nix flake check $flake6Dir +nix registry pin flake1 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 5 ]] # Test 'nix flake clone'. rm -rf $TEST_ROOT/flake1-v2 @@ -550,6 +473,10 @@ nix flake lock $flake3Dir [[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["flake2"]' ]] # Test overriding inputs of inputs. +writeTrivialFlake $flake7Dir +git -C $flake7Dir add flake.nix +git -C $flake7Dir commit -m 'Initial' + cat > $flake3Dir/flake.nix < $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < $templatesDir/trivial/a +echo b > $templatesDir/trivial/b + +git -C $templatesDir add flake.nix trivial/ +git -C $templatesDir commit -m 'Initial' + +nix flake check templates +nix flake show templates +nix flake show templates --json | jq + +createGitRepo $flakeDir +(cd $flakeDir && nix flake init) +(cd $flakeDir && nix flake init) # check idempotence +git -C $flakeDir add flake.nix +nix flake check $flakeDir +nix flake show $flakeDir +nix flake show $flakeDir --json | jq +git -C $flakeDir commit -a -m 'Initial' + +# Test 'nix flake init' with benign conflicts +createGitRepo "$flakeDir" +echo a > $flakeDir/a +(cd $flakeDir && nix flake init) # check idempotence + +# Test 'nix flake init' with conflicts +createGitRepo "$flakeDir" +echo b > $flakeDir/a +pushd $flakeDir +(! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" +popd +git -C $flakeDir commit -a -m 'Changed' + +# Test 'nix flake new'. +rm -rf $flakeDir +nix flake new -t templates#trivial $flakeDir +nix flake new -t templates#trivial $flakeDir # check idempotence +nix flake check $flakeDir diff --git a/tests/local.mk b/tests/local.mk index 8f266a330..9e372094f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,6 +3,7 @@ nix_tests = \ flakes/run.sh \ flakes/mercurial.sh \ flakes/circular.sh \ + flakes/init.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ From d16f1070f42be987093e2252c1e7bbebe0565ae0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 20:46:22 +0200 Subject: [PATCH 095/119] Split off following paths tests --- tests/flakes/flakes.sh | 148 +--------------------------------- tests/flakes/follow-paths.sh | 150 +++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 152 insertions(+), 147 deletions(-) create mode 100644 tests/flakes/follow-paths.sh diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index c99e828f3..67f00a3e5 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -13,13 +13,8 @@ flake7Dir=$TEST_ROOT/flake7 nonFlakeDir=$TEST_ROOT/nonFlake badFlakeDir=$TEST_ROOT/badFlake flakeGitBare=$TEST_ROOT/flakeGitBare -flakeFollowsA=$TEST_ROOT/follows/flakeA -flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB -flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC -flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD -flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE -for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir $flakeFollowsA; do +for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do # Give one repo a non-main initial branch. extraArgs= if [[ $repo == $flake2Dir ]]; then @@ -551,127 +546,6 @@ nix flake lock $flake3Dir --update-input flake2/flake1 # Test 'nix flake metadata --json'. nix flake metadata $flake3Dir --json | jq . -# Test flake follow paths -mkdir -p $flakeFollowsB -mkdir -p $flakeFollowsC -mkdir -p $flakeFollowsD -mkdir -p $flakeFollowsE - -cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' - # Test flake in store does not evaluate rm -rf $badFlakeDir mkdir $badFlakeDir @@ -680,23 +554,3 @@ nix store delete $(nix store add-path $badFlakeDir) [[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] [[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] - -# Non-existant follows causes an error - -cat >$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" -nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh new file mode 100644 index 000000000..19cc1bafa --- /dev/null +++ b/tests/flakes/follow-paths.sh @@ -0,0 +1,150 @@ +source ./common.sh + +requireGit + +flakeFollowsA=$TEST_ROOT/follows/flakeA +flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB +flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC +flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD +flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE + +# Test following path flakerefs. +createGitRepo $flakeFollowsA +mkdir -p $flakeFollowsB +mkdir -p $flakeFollowsC +mkdir -p $flakeFollowsD +mkdir -p $flakeFollowsE + +cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' + +# Non-existant follows should print a warning. +cat >$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" +nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" diff --git a/tests/local.mk b/tests/local.mk index 9e372094f..1540a1c8c 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -4,6 +4,7 @@ nix_tests = \ flakes/mercurial.sh \ flakes/circular.sh \ flakes/init.sh \ + flakes/follow-paths.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ From 7abcafcfeacad587ccf1a7e58334958e25cef5b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 20:49:07 +0200 Subject: [PATCH 096/119] Move the 'nix bundle' tests Note: these were previously not actually called. --- tests/{flake-bundler.sh => flakes/bundle.sh} | 9 ++------- tests/local.mk | 1 + 2 files changed, 3 insertions(+), 7 deletions(-) rename tests/{flake-bundler.sh => flakes/bundle.sh} (84%) diff --git a/tests/flake-bundler.sh b/tests/flakes/bundle.sh similarity index 84% rename from tests/flake-bundler.sh rename to tests/flakes/bundle.sh index 9496b8f92..67bbb05ac 100644 --- a/tests/flake-bundler.sh +++ b/tests/flakes/bundle.sh @@ -1,9 +1,6 @@ source common.sh -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local - -cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME +cp ../simple.nix ../simple.builder.sh ../config.nix $TEST_HOME cd $TEST_HOME @@ -25,6 +22,7 @@ cat < flake.nix }; } EOF + nix build .# nix bundle --bundler .# .# nix bundle --bundler .#bundlers.$system.default .#packages.$system.default @@ -32,6 +30,3 @@ nix bundle --bundler .#bundlers.$system.simple .#packages.$system.default nix bundle --bundler .#bundlers.$system.default .#apps.$system.default nix bundle --bundler .#bundlers.$system.simple .#apps.$system.default - -clearStore - diff --git a/tests/local.mk b/tests/local.mk index 1540a1c8c..23eb92039 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -5,6 +5,7 @@ nix_tests = \ flakes/circular.sh \ flakes/init.sh \ flakes/follow-paths.sh \ + flakes/bundle.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ From 6ba45f81a8a8fcf404f899b9f6879dd6af7d8ac4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 20:51:28 +0200 Subject: [PATCH 097/119] Move flake-local-settings.sh --- tests/{flake-local-settings.sh => flakes/config.sh} | 5 +---- tests/local.mk | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) rename tests/{flake-local-settings.sh => flakes/config.sh} (88%) diff --git a/tests/flake-local-settings.sh b/tests/flakes/config.sh similarity index 88% rename from tests/flake-local-settings.sh rename to tests/flakes/config.sh index e92c16f87..d1941a6be 100644 --- a/tests/flake-local-settings.sh +++ b/tests/flakes/config.sh @@ -1,9 +1,6 @@ source common.sh -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local - -cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME +cp ../simple.nix ../simple.builder.sh ../config.nix $TEST_HOME cd $TEST_HOME diff --git a/tests/local.mk b/tests/local.mk index 23eb92039..2b5b6858e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -85,7 +85,7 @@ nix_tests = \ nix-copy-ssh.sh \ post-hook.sh \ function-trace.sh \ - flake-local-settings.sh \ + flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ From 752158a8efb7a027e5020d3c9c03aa5d70002b56 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 20:55:08 +0200 Subject: [PATCH 098/119] Move flake-searching.sh and make it less dependent on git --- .../search-root.sh} | 24 +++++++++---------- tests/local.mk | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) rename tests/{flake-searching.sh => flakes/search-root.sh} (72%) diff --git a/tests/flake-searching.sh b/tests/flakes/search-root.sh similarity index 72% rename from tests/flake-searching.sh rename to tests/flakes/search-root.sh index db241f6d2..d8586dc8a 100644 --- a/tests/flake-searching.sh +++ b/tests/flakes/search-root.sh @@ -1,15 +1,11 @@ source common.sh -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping flake search tests" - exit 99 -fi - clearStore -cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME +writeSimpleFlake $TEST_HOME cd $TEST_HOME mkdir -p foo/subdir + echo '{ outputs = _: {}; }' > foo/flake.nix cat < flake.nix { @@ -43,10 +39,12 @@ nix build --override-input foo . || fail "flake should search up directories whe sed "s,$PWD/foo,$PWD/foo/subdir,g" -i flake.nix ! nix build || fail "flake should not search upwards when part of inputs" -pushd subdir -git init -for i in "${success[@]}" "${failure[@]}"; do - ! nix build $i || fail "flake should not search past a git repository" -done -rm -rf .git -popd +if [[ -n $(type -p git) ]]; then + pushd subdir + git init + for i in "${success[@]}" "${failure[@]}"; do + ! nix build $i || fail "flake should not search past a git repository" + done + rm -rf .git + popd +fi diff --git a/tests/local.mk b/tests/local.mk index 2b5b6858e..25d23a2fc 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -49,7 +49,7 @@ nix_tests = \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ - flake-searching.sh \ + flakes/search-root.sh \ ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ From b15c4fdbde738310188be035807de7d5b2036a28 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jul 2022 21:01:16 +0200 Subject: [PATCH 099/119] Split off 'nix flake check' tests --- tests/flakes/check.sh | 89 ++++++++++++++++++++++++++++++++++++++++++ tests/flakes/flakes.sh | 86 ---------------------------------------- tests/local.mk | 1 + 3 files changed, 90 insertions(+), 86 deletions(-) create mode 100644 tests/flakes/check.sh diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh new file mode 100644 index 000000000..f572aa75c --- /dev/null +++ b/tests/flakes/check.sh @@ -0,0 +1,89 @@ +source common.sh + +flakeDir=$TEST_ROOT/flake3 +mkdir -p $flakeDir + +cat > $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check should have failed" || true) +echo "$checkRes" | grep -q "packages.system-1.default" +echo "$checkRes" | grep -q "packages.system-2.default" diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index 67f00a3e5..267e2cd6f 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -340,92 +340,6 @@ rm -rf $TEST_ROOT/flake1-v2 nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 [ -e $TEST_ROOT/flake1-v2/flake.nix ] -# More 'nix flake check' tests. -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix <&1 && fail "nix flake check should have failed" || true) -echo "$checkRes" | grep -q "packages.system-1.default" -echo "$checkRes" | grep -q "packages.system-2.default" - # Test 'follows' inputs. cat > $flake3Dir/flake.nix < Date: Thu, 14 Jul 2022 15:06:11 +0200 Subject: [PATCH 100/119] On test failures, print a bash stack trace This makes it easier to identify what command failed. It looks like: follow-paths.sh: test failed at: main in follow-paths.sh:54 --- tests/common.sh.in | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/common.sh.in b/tests/common.sh.in index 5efd025ee..79da10199 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -119,11 +119,11 @@ killDaemon() { } restartDaemon() { - [[ -z "${pidDaemon:-}" ]] && return 0 + [[ -z "${pidDaemon:-}" ]] && return 0 - killDaemon - unset NIX_REMOTE - startDaemon + killDaemon + unset NIX_REMOTE + startDaemon } if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then @@ -190,4 +190,15 @@ if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then startDaemon fi +onError() { + set +x + echo "$0: test failed at:" >&2 + for ((i = 1; i < 16; i++)); do + if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi + echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 + done +} + +trap onError ERR + fi # COMMON_SH_SOURCED From ff49c75502845d9938f3a479f1696ee30d3b45b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Jul 2022 17:46:26 +0200 Subject: [PATCH 101/119] Disable auto-chroot if $NIX_STORE_DIR is set Fixes #6732. --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 05353bce2..45c53f23e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1321,7 +1321,7 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); #if __linux__ - else if (!pathExists(stateDir) && params.empty() && getuid() != 0) { + else if (!pathExists(stateDir) && params.empty() && getuid() != 0 && !getEnv("NIX_STORE_DIR").has_value()) { /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ From 99208bb8ccd7387c09907e7b17d0091f2f767b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 14 Jul 2022 17:36:31 -0500 Subject: [PATCH 102/119] curl: patch for netrc regression in Nix --- flake.lock | 6 +++--- flake.nix | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 01e4f506a..a66c9cb1b 100644 --- a/flake.lock +++ b/flake.lock @@ -18,11 +18,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1653988320, - "narHash": "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c=", + "lastModified": 1657693803, + "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2fa57ed190fd6c7c746319444f34b5917666e5c1", + "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1ee040441..1b26460e7 100644 --- a/flake.nix +++ b/flake.nix @@ -108,7 +108,7 @@ ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; buildDeps = - [ curl + [ (curl.override { patchNetrcRegression = true; }) bzip2 xz brotli editline openssl sqlite libarchive @@ -363,7 +363,7 @@ buildInputs = [ nix - curl + (curl.override { patchNetrcRegression = true; }) bzip2 xz pkgs.perl From 04386f7d69d9c370eb4367ca41d89ac5990ac02e Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Thu, 14 Jul 2022 23:11:02 -0700 Subject: [PATCH 103/119] nix develop: do not assume that saved vars are set This fixes https://github.com/NixOS/nix/issues/6809 --- src/nix/develop.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6d9ad9942..ba7ba7c25 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -288,8 +288,10 @@ struct Common : InstallableCommand, MixProfile out << "unset shellHook\n"; - for (auto & var : savedVars) + for (auto & var : savedVars) { + out << fmt("%s=${%s:-}\n", var, var); out << fmt("nix_saved_%s=\"$%s\"\n", var, var); + } buildEnvironment.toBash(out, ignoreVars); From 3bcd7a5474f065a6e94a60b3042a0d42ed0184ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 15 Jul 2022 12:32:29 +0200 Subject: [PATCH 104/119] Disable auto-chroot if $NIX_STATE_DIR is set Issue #6732. --- src/libstore/store-api.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 45c53f23e..91dbd991e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1321,7 +1321,12 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); #if __linux__ - else if (!pathExists(stateDir) && params.empty() && getuid() != 0 && !getEnv("NIX_STORE_DIR").has_value()) { + else if (!pathExists(stateDir) + && params.empty() + && getuid() != 0 + && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) + { /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ From b88fb50e218cd3099cbceace48f7cfdf50a8f11f Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Wed, 6 Jul 2022 11:28:23 -0400 Subject: [PATCH 105/119] fix(libstore): allow Nix to access all Rosetta 2 paths on MacOS Fixes: #5884 --- src/libstore/sandbox-defaults.sb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 56b35c3fe..d9d710559 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -98,7 +98,9 @@ (allow file* (literal "/private/var/select/sh")) -; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. +; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin (and vice versa). (allow file-read* (subpath "/Library/Apple/usr/libexec/oah") - (subpath "/System/Library/Apple/usr/libexec/oah")) + (subpath "/System/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") + (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) From 8ea3a911aa81d41efdff231f4b42b11d8861a000 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 14:39:22 -0700 Subject: [PATCH 106/119] local-derivation-goal.cc: improve error messages when sandboxing fails The failure modes for nix's sandboxing setup are pretty complicated. When nix is unable to set up the sandbox, let's provide more detail about what went wrong. Specifically: * Make sure the error message includes the word "sandbox" so the user knows that the failure was related to sandboxing. * If `--option sandbox-fallback false` was provided, and removing it would have allowed further attempts to make progress, let the user know. --- src/libstore/build/local-derivation-goal.cc | 24 +++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d1ec91ed5..86a59e427 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -850,13 +850,23 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - + if (child == -1) + switch(errno) { + case EPERM: + case EINVAL: { + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); + /* Mention sandbox-fallback in the error message so the user + knows that having it disabled contributed to the + unrecoverability of this failure */ + throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + } + default: + throw SysError("creating sandboxed builder process using clone()"); + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); From 90830b1074cd09b58adde859fb1741a33390412f Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:28:13 -0700 Subject: [PATCH 107/119] local-derivation-goal.cc: warn if failing due to max_user_namespaces==0 This commit uses `warn()` to notify the user if sandbox setup fails with errno==EPERM and /proc/sys/user/max_user_namespaces is missing or zero, since that is at least part of the reason why sandbox setup failed. Note that `echo -n 0 > /proc/sys/user/max_user_namespaces` or equivalent at boot time has been the recommended mitigation for several Linux LPE vulnerabilities over the past few years. Many users have applied this mitigation and then forgotten that they have done so. --- src/libstore/build/local-derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 86a59e427..674b2eaa3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -859,6 +859,8 @@ void LocalDerivationGoal::startBuilder() to true (the default). */ if (settings.sandboxFallback) _exit(1); + if (!userNamespacesEnabled && errno==EPERM) + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 8d35f387dcfa61c59f898de88ef45f3f97817267 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:37:27 -0700 Subject: [PATCH 108/119] local-derivation-goal.cc: warn if failing and /proc/self/ns/user missing This commit causes nix to `warn()` if sandbox setup has failed and `/proc/self/ns/user` does not exist. This is usually a sign that the kernel was compiled without `CONFIG_USER_NS=y`, which is required for sandboxing. --- src/libstore/build/local-derivation-goal.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 674b2eaa3..3aa85e264 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,6 +861,9 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + Path procSelfNsUser = "/proc/self/ns/user"; + if (!pathExists(procSelfNsUser)) + warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 6fc56318bf32f715de8634c199c0fb812f813a8c Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:23:27 -0700 Subject: [PATCH 109/119] local-derivation-goal.cc: add comment re: CLONE_NEWUSER local-derivation-goal.cc contains a comment stating that "Some distros patch Linux to not allow unprivileged user namespaces." Let's give a pointer to a common version of this patch for those who want more details about this failure mode. --- src/libstore/build/local-derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3aa85e264..1c7618045 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -845,6 +845,7 @@ void LocalDerivationGoal::startBuilder() /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. + * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 */ usingUserNamespace = false; flags &= ~CLONE_NEWUSER; From c8c6203c2c6912a52c5f08881c453a04e7fc3f58 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:27:22 -0700 Subject: [PATCH 110/119] local-derivation-goal.cc: detect unprivileged_userns_clone failure mode The workaround for "Some distros patch Linux" mentioned in local-derivation-goal.cc will not help in the `--option sandbox-fallback false` case. To provide the user more helpful guidance on how to get the sandbox working, let's check to see if the `/proc` node created by the aforementioned patch is present and configured in a way that will cause us problems. If so, give the user a suggestion for how to troubleshoot the problem. --- src/libstore/build/local-derivation-goal.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 1c7618045..047c5c8ea 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -862,6 +862,13 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + if (userNamespacesEnabled) { + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + } + } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); From 5f51539f88227285866843f1383fd47d80fd5918 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:30:52 -0700 Subject: [PATCH 111/119] change warn() to notice() --- src/libstore/build/local-derivation-goal.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 047c5c8ea..595149f0a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,17 +861,17 @@ void LocalDerivationGoal::startBuilder() if (settings.sandboxFallback) _exit(1); if (!userNamespacesEnabled && errno==EPERM) - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); } } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) - warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 99fcc91f67ece5a9646065665395f496d6a0cb84 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:33:12 -0700 Subject: [PATCH 112/119] as requested by @thufschmitt https://github.com/NixOS/nix/pull/6814#discussion_r924275777 --- src/libstore/build/local-derivation-goal.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 595149f0a..43df41e34 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -855,11 +855,6 @@ void LocalDerivationGoal::startBuilder() switch(errno) { case EPERM: case EINVAL: { - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -872,6 +867,11 @@ void LocalDerivationGoal::startBuilder() Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From a9e75eca00f7efe361545c6e77ecb65244230dc3 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:33 -0700 Subject: [PATCH 113/119] error.hh: add additional constructor with explicit errno argument --- src/libutil/error.hh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a53e9802e..3d1479c54 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -204,13 +204,19 @@ public: int errNo; template - SysError(const Args & ... args) + SysError(int errNo_, const Args & ... args) : Error("") { - errNo = errno; + errNo = errNo_; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } + + template + SysError(const Args & ... args) + : SysError(errno, args ...) + { + } }; } From 36e1383b6b32abd414c8402dd66f7ef78f93f4e1 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:52 -0700 Subject: [PATCH 114/119] local-derivation-goal.cc: save global errno to the stack before performing tests which might clobber it --- src/libstore/build/local-derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 43df41e34..79a241ae0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -851,10 +851,11 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - if (child == -1) + if (child == -1) { switch(errno) { case EPERM: case EINVAL: { + int errno_ = errno; if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -875,11 +876,12 @@ void LocalDerivationGoal::startBuilder() /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ - throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); } default: throw SysError("creating sandboxed builder process using clone()"); } + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); From 1af5d798a4d10a07e995d420a759f2fe752b583a Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Fri, 15 Jul 2022 00:14:29 -0400 Subject: [PATCH 115/119] libstore/globals.cc: Automatically set cores based on cgroup CPU limit By default, Nix sets the "cores" setting to the number of CPUs which are physically present on the machine. If cgroups are used to limit the CPU and memory consumption of a large Nix build, the OOM killer may be invoked. For example, consider a GitLab CI pipeline which builds a large software package. The GitLab runner spawns a container whose CPU is limited to 4 cores and whose memory is limited to 16 GiB. If the underlying machine has 64 cores, Nix will invoke the build with -j64. In many cases, that level of parallelism will invoke the OOM killer and the build will completely fail. This change sets the default value of "cores" to be ceil(cpu_quota / cpu_period), with a fallback to std::thread::hardware_concurrency() if cgroups v2 is not detected. --- src/libstore/globals.cc | 50 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0f2ca4b15..48df839fa 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,6 +11,11 @@ #include #include +#if __linux__ +#include +#include +#endif + #include @@ -114,7 +119,50 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - return std::max(1U, std::thread::hardware_concurrency()); + unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + + #if __linux__ + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return concurrency; + + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + concurrency = std::ceil(std::stoi(quota) / std::stof(period)); + } + } + #endif + + return concurrency; } StringSet Settings::getDefaultSystemFeatures() From 722de8ddcc875c7e8e9a228f9d88454bae31fd40 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 19 Jul 2022 02:09:46 -0400 Subject: [PATCH 116/119] libstore/globals.cc: Move cgroup detection to libutil --- src/libstore/globals.cc | 54 +++++------------------------------------ src/libutil/util.cc | 51 ++++++++++++++++++++++++++++++++++++++ src/libutil/util.hh | 3 +++ 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 48df839fa..d724897bb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,11 +11,6 @@ #include #include -#if __linux__ -#include -#include -#endif - #include @@ -119,50 +114,13 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int maxCPU = getMaxCPU(); - #if __linux__ - FILE *fp = fopen("/proc/mounts", "r"); - if (!fp) - return concurrency; - - Strings cgPathParts; - - struct mntent *ent; - while ((ent = getmntent(fp))) { - std::string mountType, mountPath; - - mountType = ent->mnt_type; - mountPath = ent->mnt_dir; - - if (mountType == "cgroup2") { - cgPathParts.push_back(mountPath); - break; - } - } - - fclose(fp); - - if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { - std::string currentCgroup = readFile("/proc/self/cgroup"); - Strings cgValues = tokenizeString(currentCgroup, ":"); - cgPathParts.push_back(trim(cgValues.back(), "\n")); - cgPathParts.push_back("cpu.max"); - std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); - - if (pathExists(fullCgPath)) { - std::string cpuMax = readFile(fullCgPath); - std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); - std::string quota = cpuMaxParts[0]; - std::string period = trim(cpuMaxParts[1], "\n"); - - if (quota != "max") - concurrency = std::ceil(std::stoi(quota) / std::stof(period)); - } - } - #endif - - return concurrency; + if (maxCPU > 0) + return maxCPU; + else + return concurrency; } StringSet Settings::getDefaultSystemFeatures() diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 28df30fef..be6fe091f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -35,6 +35,9 @@ #ifdef __linux__ #include #include + +#include +#include #endif @@ -788,7 +791,55 @@ void drainFD(int fd, Sink & sink, bool block) } } +////////////////////////////////////////////////////////////////////// +unsigned int getMaxCPU() +{ + #if __linux__ + try { + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return 0; + + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } + } + } catch (Error &) { ignoreException(); } + #endif + + return 0; +} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d3ed15b0b..29227ecc6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -182,6 +182,9 @@ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); +/* If cgroups are active, attempt to calculate the number of CPUs available. + If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +unsigned int getMaxCPU(); /* Automatic cleanup of resources. */ From 228028fc1aaad20a217387fbe9d4aa2d8698a048 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Thu, 28 Jul 2022 03:36:39 -0400 Subject: [PATCH 117/119] docker.nix: Allow Nix configuration to be customized --- docker.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index ddf6feff5..8e6aa227f 100644 --- a/docker.nix +++ b/docker.nix @@ -6,6 +6,7 @@ , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" , extraPkgs ? [] , maxLayers ? 100 +, nixConf ? {} }: let defaultPkgs = with pkgs; [ @@ -123,12 +124,17 @@ let (lib.attrValues (lib.mapAttrs groupToGroup groups)) ); - nixConf = { + defaultNixConf = { sandbox = "false"; build-users-group = "nixbld"; - trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; }; - nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: + let + vStr = if builtins.isList v then lib.concatStringsSep " " v else v; + in + "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n"; baseSystem = let From be4654c344f0745d4c2eefb33a878bd1f23f6b40 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 15:55:41 +0200 Subject: [PATCH 118/119] manual: fix section title in table of contents --- doc/manual/src/SUMMARY.md.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9728728aa..5df4e2d75 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -33,7 +33,7 @@ - [Arguments and Variables](expressions/arguments-variables.md) - [Building and Testing](expressions/simple-building-testing.md) - [Generic Builder Syntax](expressions/generic-builder.md) - - [Writing Nix Expressions](expressions/expression-language.md) + - [Nix Expression Language](expressions/expression-language.md) - [Values](expressions/language-values.md) - [Language Constructs](expressions/language-constructs.md) - [Operators](expressions/language-operators.md) From 85cdaebcd69c4f6abd15b77888f0093be9c5ada4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 16:10:22 +0200 Subject: [PATCH 119/119] manual: set -> attribute set reword description to have shorter sentences. --- doc/manual/src/expressions/language-values.md | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 75ae9f2eb..abbe1fd35 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -172,25 +172,27 @@ function and the fifth being a set. Note that lists are only lazy in values, and they are strict in length. -## Sets +## Attribute Sets -Sets are really the core of the language, since ultimately the Nix -language is all about creating derivations, which are really just sets -of attributes to be passed to build scripts. +Attribute sets are collections of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). -Sets are just a list of name/value pairs (called *attributes*) enclosed -in curly brackets, where each value is an arbitrary expression -terminated by a semicolon. For example: +Names and values are separated by an equal sign (`=`). +Each value is an arbitrary expression terminated by a semicolon (`;`). + +Attributes can appear in any order. +An attribute name may only occur once. + +Example: ```nix -{ x = 123; +{ + x = 123; text = "Hello"; y = f { bla = 456; }; } ``` -This defines a set with attributes named `x`, `text`, `y`. The order of -the attributes is irrelevant. An attribute name may only occur once. +This defines a set with attributes named `x`, `text`, `y`. Attributes can be selected from a set using the `.` operator. For instance,